From fc0346a7908dee49d39d090d9b2cc27af762f3a5 Mon Sep 17 00:00:00 2001 From: OMGeeky Date: Tue, 24 Jan 2023 18:18:42 +0100 Subject: [PATCH] backup --- .gitignore | 2 + .idea/.gitignore | 8 +++ .idea/exponential_backoff.iml | 11 ++++ .idea/misc.xml | 6 ++ .idea/modules.xml | 8 +++ .idea/vcs.xml | 6 ++ Cargo.toml | 15 +++++ src/bigquery.rs | 9 +++ src/lib.rs | 28 +++++++++ src/twitch.rs | 104 ++++++++++++++++++++++++++++++++++ 10 files changed, 197 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/exponential_backoff.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 Cargo.toml create mode 100644 src/bigquery.rs create mode 100644 src/lib.rs create mode 100644 src/twitch.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4fffb2f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/exponential_backoff.iml b/.idea/exponential_backoff.iml new file mode 100644 index 0000000..c254557 --- /dev/null +++ b/.idea/exponential_backoff.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..3ce3588 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..4904539 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e83e126 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "exponential_backoff" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = "1.24.2" +reqwest = "0.11.14" +chrono = "0.4.23" +google-youtube3 = "4.0" +mime = "0.2.6" +serde_json = "1.0.91" +rand = "0.8.5" \ No newline at end of file diff --git a/src/bigquery.rs b/src/bigquery.rs new file mode 100644 index 0000000..5252774 --- /dev/null +++ b/src/bigquery.rs @@ -0,0 +1,9 @@ + +use std::error::Error; + + +// pub async fn check_backoff_bigquery(res: Result> )->Result>{ +// +// +// Ok(res) +// } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..44a83fa --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,28 @@ +#![feature(async_closure)] +use rand::Rng; + +const EXTRA_BUFFER_TIME: u64 = 100; + +pub enum Api { + Youtube, + Twitch, + Bigquery, +} + +pub mod errors; +pub mod youtube; +pub mod twitch; +pub mod bigquery; + + +async fn sleep_for_backoff_time(backoff_time: u64, with_extra_buffer_time: bool) { + let extra_buffer_time = match with_extra_buffer_time { + true => EXTRA_BUFFER_TIME, + false => 0 + }; + let backoff_time = backoff_time * 1000 as u64; + + // let random_extra = rand::thread_rng().gen_range(0..100); + let random_extra = rand::thread_rng().gen_range(0..100); + tokio::time::sleep(std::time::Duration::from_millis(backoff_time + extra_buffer_time + random_extra)).await; +} diff --git a/src/twitch.rs b/src/twitch.rs new file mode 100644 index 0000000..ca76297 --- /dev/null +++ b/src/twitch.rs @@ -0,0 +1,104 @@ +use std::error::Error; + +use chrono::NaiveDateTime; +use reqwest::{Body, Client, IntoUrl, Request, RequestBuilder, Response}; +use reqwest::header::{HeaderMap, HeaderValue}; + +use crate::errors::BackoffError; +use crate::sleep_for_backoff_time; + +enum ErrorTypes { + E429(String), + +} +//region reqwest +//region convenience functions + +//region GET + +pub async fn check_backoff_twitch_get(url: T) -> Result> { + check_backoff_twitch(Request::new(reqwest::Method::GET, url.into_url()?)).await +} + +pub async fn check_backoff_twitch_get_with_client(url: T, client: &Client) -> Result> { + check_backoff_twitch_with_client(Request::new(reqwest::Method::GET, url.into_url()?), client).await +} + +//endregion + +//region POST + +pub async fn check_backoff_twitch_post>(url: T, headers: Option, body: Option) -> Result> { + let client = Client::new(); + check_backoff_twitch_post_with_client(url, headers, body, &client).await +} + +pub async fn check_backoff_twitch_post_with_client>( + url: T, + headers: Option, + body: Option, + client: &Client +) -> Result> { + let mut request = client.post(url.into_url()?); + + if let Some(headers) = headers { + request = request.headers(headers); + } + if let Some(body) = body { + request = request.body(body); + } + + let request = request.build()?; + check_backoff_twitch_with_client(request, client).await +} +//endregion + +pub async fn check_backoff_twitch(request: Request) -> Result> { + let client = Client::new(); + check_backoff_twitch_with_client(request, &client).await +} + +//endregion + +pub async fn check_backoff_twitch_with_client(request: Request, client: &Client) -> Result> { + loop { + let r: Request = request.try_clone().ok_or::("Request is None".into())?; + // Some(v) => Ok(v), + // None => Err("Request is None".into()), + // }?; + let response = client.execute(r).await?; + + let status_code = response.status(); + match status_code.as_u16() { + 200 => return Ok(response), + 429 => { + let x = &request.headers().get("Ratelimit-Reset").ok_or(BackoffError::new("No rate limit reset given"))?; + let value: String = x.to_str()?.to_string(); + handle_e429(value).await?; + }, + + _ => return Ok(response), + // _ => todo!("Handle other errors or "), + } + } +} + +async fn handle_e429(value: String) -> Result<(), Box> { + let value = value.parse::()?; + let timestamp = NaiveDateTime::from_timestamp_opt(value, 0) + .ok_or(BackoffError::new(format!("Could not convert the provided timestamp: {}", value)))?; + let now = chrono::Local::now().naive_local(); + if timestamp < now { + sleep_for_backoff_time(1000, true).await; + return Ok(()); + } + let duration = timestamp - now; + let duration = duration.num_milliseconds() as u64; + println!("Sleeping for {} seconds", duration); + sleep_for_backoff_time(duration, true).await; + // tokio::time::sleep(tokio::time::Duration::from_secs(duration)).await; + //TODO: test this somehow + Ok(()) +} + +//endregion