OMGeeky/exponential_backoff
1
0
Fork 0
mirror of https://github.com/OMGeeky/exponential_backoff.git synced 2026-02-23 15:38:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

backup

Browse Source
This commit is contained in:
OMGeeky
2023-01-24 18:18:42 +01:00
parent d7e99115f3
commit fc0346a790
10 changed files with 197 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/target
/Cargo.lock

8
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

11
.idea/exponential_backoff.iml generated Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="CPP_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MarkdownSettingsMigration">
<option name="stateVersion" value="1" />
</component>
</project>

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/exponential_backoff.iml" filepath="$PROJECT_DIR$/.idea/exponential_backoff.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

15
Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "exponential_backoff"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = "1.24.2"
reqwest = "0.11.14"
chrono = "0.4.23"
google-youtube3 = "4.0"
mime = "0.2.6"
serde_json = "1.0.91"
rand = "0.8.5"

9
src/bigquery.rs Normal file
View File

@@ -0,0 +1,9 @@
use std::error::Error;
// pub async fn check_backoff_bigquery<T>(res: Result<T, Box<dyn Error>> )->Result<T, Box<dyn Error>>{
//
//
// Ok(res)
// }

28
src/lib.rs Normal file
View File

@@ -0,0 +1,28 @@
#![feature(async_closure)]
use rand::Rng;
const EXTRA_BUFFER_TIME: u64 = 100;
pub enum Api {
Youtube,
Twitch,
Bigquery,
}
pub mod errors;
pub mod youtube;
pub mod twitch;
pub mod bigquery;
async fn sleep_for_backoff_time(backoff_time: u64, with_extra_buffer_time: bool) {
let extra_buffer_time = match with_extra_buffer_time {
true => EXTRA_BUFFER_TIME,
false => 0
};
let backoff_time = backoff_time * 1000 as u64;
// let random_extra = rand::thread_rng().gen_range(0..100);
let random_extra = rand::thread_rng().gen_range(0..100);
tokio::time::sleep(std::time::Duration::from_millis(backoff_time + extra_buffer_time + random_extra)).await;
}

104
src/twitch.rs Normal file
View File

@@ -0,0 +1,104 @@
use std::error::Error;
use chrono::NaiveDateTime;
use reqwest::{Body, Client, IntoUrl, Request, RequestBuilder, Response};
use reqwest::header::{HeaderMap, HeaderValue};
use crate::errors::BackoffError;
use crate::sleep_for_backoff_time;
enum ErrorTypes {
E429(String),
}
//region reqwest
//region convenience functions
//region GET
pub async fn check_backoff_twitch_get<T: IntoUrl>(url: T) -> Result<Response, Box<dyn Error>> {
check_backoff_twitch(Request::new(reqwest::Method::GET, url.into_url()?)).await
}
pub async fn check_backoff_twitch_get_with_client<T: IntoUrl>(url: T, client: &Client) -> Result<Response, Box<dyn Error>> {
check_backoff_twitch_with_client(Request::new(reqwest::Method::GET, url.into_url()?), client).await
}
//endregion
//region POST
pub async fn check_backoff_twitch_post<T: IntoUrl, B: Into<Body>>(url: T, headers: Option<HeaderMap>, body: Option<B>) -> Result<Response, Box<dyn Error>> {
let client = Client::new();
check_backoff_twitch_post_with_client(url, headers, body, &client).await
}
pub async fn check_backoff_twitch_post_with_client<T: IntoUrl, B: Into<Body>>(
url: T,
headers: Option<HeaderMap>,
body: Option<B>,
client: &Client
) -> Result<Response, Box<dyn Error>> {
let mut request = client.post(url.into_url()?);
if let Some(headers) = headers {
request = request.headers(headers);
}
if let Some(body) = body {
request = request.body(body);
}
let request = request.build()?;
check_backoff_twitch_with_client(request, client).await
}
//endregion
pub async fn check_backoff_twitch(request: Request) -> Result<Response, Box<dyn Error>> {
let client = Client::new();
check_backoff_twitch_with_client(request, &client).await
}
//endregion
pub async fn check_backoff_twitch_with_client(request: Request, client: &Client) -> Result<Response, Box<dyn Error>> {
loop {
let r: Request = request.try_clone().ok_or::<BackoffError>("Request is None".into())?;
// Some(v) => Ok(v),
// None => Err("Request is None".into()),
// }?;
let response = client.execute(r).await?;
let status_code = response.status();
match status_code.as_u16() {
200 => return Ok(response),
429 => {
let x = &request.headers().get("Ratelimit-Reset").ok_or(BackoffError::new("No rate limit reset given"))?;
let value: String = x.to_str()?.to_string();
handle_e429(value).await?;
},
_ => return Ok(response),
// _ => todo!("Handle other errors or "),
}
}
}
async fn handle_e429(value: String) -> Result<(), Box<dyn Error>> {
let value = value.parse::<i64>()?;
let timestamp = NaiveDateTime::from_timestamp_opt(value, 0)
.ok_or(BackoffError::new(format!("Could not convert the provided timestamp: {}", value)))?;
let now = chrono::Local::now().naive_local();
if timestamp < now {
sleep_for_backoff_time(1000, true).await;
return Ok(());
}
let duration = timestamp - now;
let duration = duration.num_milliseconds() as u64;
println!("Sleeping for {} seconds", duration);
sleep_for_backoff_time(duration, true).await;
// tokio::time::sleep(tokio::time::Duration::from_secs(duration)).await;
//TODO: test this somehow
Ok(())
}
//endregion