OMGeeky/exponential_backoff
1
0
Fork 0
mirror of https://github.com/OMGeeky/exponential_backoff.git synced 2026-01-03 01:56:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

logging

Browse Source
This commit is contained in:
OMGeeky
2023-04-04 18:08:50 +02:00
parent 986a9dba83
commit 714b882666
6 changed files with 36 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@@ -12,3 +12,4 @@ chrono = "0.4.23"
google-youtube3 = "4.0"
serde_json = "1.0.91"
rand = "0.8.5"
log = "0.4.17"

2
src/bigquery.rs
View File

@@ -1,3 +1 @@
use std::error::Error;
//TODO: implement bigquery backoff

3
src/errors.rs
View File

@@ -1,5 +1,7 @@
use std::fmt;
use log::warn;
#[derive(Debug, Clone)]
pub struct BackoffError {
message: String,
@@ -8,6 +10,7 @@ pub struct BackoffError {
impl BackoffError {
pub fn new<S: Into<String>>(message: S) -> BackoffError {
let message = message.into();
warn!("BackoffError: {}", message);
BackoffError { message }
}
}

9
src/lib.rs
View File

@@ -1,5 +1,7 @@
use rand::Rng;
use log::{trace, info};
const EXTRA_BUFFER_TIME: u64 = 100;
pub enum Api {
@@ -21,6 +23,11 @@ async fn sleep_for_backoff_time(backoff_time: u64, with_extra_buffer_time: bool)
true => EXTRA_BUFFER_TIME,
false => 0,
};
trace!(
"sleep_for_backoff_time->backoff_time: {}, extra_buffer_time: {}",
backoff_time,
extra_buffer_time
);
//convert to milliseconds
let backoff_time = backoff_time * 1000;
@@ -28,7 +35,7 @@ async fn sleep_for_backoff_time(backoff_time: u64, with_extra_buffer_time: bool)
//add some random extra time for good measure (in milliseconds)
let random_extra = rand::thread_rng().gen_range(0..100);
let total_millis = backoff_time + extra_buffer_time + random_extra;
println!(
info!(
"sleep_for_backoff_time->Sleeping for {} milliseconds",
total_millis
);

21
src/twitch.rs
View File

@@ -1,15 +1,14 @@
use std::error::Error;
use chrono::NaiveDateTime;
use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::{Body, Client, IntoUrl, Request, RequestBuilder, Response};
use reqwest::{Body, Client, IntoUrl, Request, Response};
use reqwest::header::HeaderMap;
use log::{info, trace};
use crate::errors::BackoffError;
use crate::sleep_for_backoff_time;
enum ErrorTypes {
E429(String),
}
//region reqwest
//region convenience functions
@@ -46,7 +45,9 @@ pub async fn check_backoff_twitch_post_with_client<T: IntoUrl, B: Into<Body>>(
body: Option<B>,
client: &Client,
) -> Result<Response, Box<dyn Error>> {
let mut request = client.post(url.into_url()?);
let url = url.into_url()?;
trace!("check_backoff_twitch_post_with_client {:?}", url);
let mut request = client.post(url);
if let Some(headers) = headers {
request = request.headers(headers);
@@ -71,7 +72,10 @@ pub async fn check_backoff_twitch_with_client(
request: Request,
client: &Client,
) -> Result<Response, Box<dyn Error>> {
trace!("check_backoff_twitch_with_client {:?}", request);
let mut counter = 0;
loop {
trace!("check_backoff_twitch_with_client loop ({})", counter);
let r: Request = request
.try_clone()
.ok_or::<BackoffError>("Request is None".into())?;
@@ -95,10 +99,13 @@ pub async fn check_backoff_twitch_with_client(
_ => return Ok(response),
// _ => todo!("Handle other errors or "),
}
counter += 1;
}
}
async fn handle_e429(value: String) -> Result<(), Box<dyn Error>> {
trace!("handle_e429 {}", value);
let value = value.parse::<i64>()?;
let timestamp = NaiveDateTime::from_timestamp_opt(value, 0).ok_or(BackoffError::new(
format!("Could not convert the provided timestamp: {}", value),
@@ -110,7 +117,7 @@ async fn handle_e429(value: String) -> Result<(), Box<dyn Error>> {
}
let duration = timestamp - now;
let duration = duration.num_seconds() as u64;
println!("Sleeping for {} seconds", duration);
info!("Sleeping for {} seconds", duration);
sleep_for_backoff_time(duration, true).await;
//TODO: test this somehow
Ok(())

16
src/youtube.rs
View File

@@ -1,8 +1,8 @@
use std::error::Error;
use std::future::Future;
use std::path::{Path, PathBuf};
use google_youtube3::api::Video;
use log::{warn, info, trace};
use google_youtube3::Error::BadRequest;
use google_youtube3::hyper::{Body, Response};
use google_youtube3::hyper::client::HttpConnector;
@@ -31,23 +31,25 @@ pub async fn generic_check_backoff_youtube<'a, 'b, 'c,
(
client: &'a YouTube<HttpsConnector<HttpConnector>>,
para: &'b Para,
function: impl Fn(&'a YouTube<HttpsConnector<HttpConnector>>, &'b Para) -> Fut
function: impl Fn(&'a YouTube<HttpsConnector<HttpConnector>>, &'b Para) -> Fut,
)
-> Result<google_youtube3::Result<(Response<Body>, T)>, Box<dyn Error>>
{
trace!("generic_check_backoff_youtube");
let mut backoff = 0;
let mut res: google_youtube3::Result<(Response<Body>, T)>;
'try_upload: loop {
trace!("generic_check_backoff_youtube loop ({})", backoff);
res = function(&client, para).await;
match res {
Ok(_) => break 'try_upload,
Err(e) => {
println!("Error: {}", e);
warn!("Error: {}", e);
if let BadRequest(e1) = &e {
let is_quota_error = get_is_quota_error(&e1);
if is_quota_error {
println!("quota_error: {}", e);
info!("quota_error: {}", e);
backoff += 1;
if !wait_for_backoff(backoff).await {
return Err(e.into());
@@ -67,8 +69,9 @@ pub async fn generic_check_backoff_youtube<'a, 'b, 'c,
}
async fn wait_for_backoff<'a>(backoff: u32) -> bool {
trace!("wait_for_backoff ({})", backoff);
let mut backoff_time = YOUTUBE_DEFAULT_BACKOFF_TIME_S.pow(backoff);
println!("backoff_time: {}", backoff_time);
info!("backoff_time: {}", backoff_time);
if backoff as u64 > YOUTUBE_MAX_TRIES {
return false;
}
@@ -112,6 +115,7 @@ async fn wait_for_backoff<'a>(backoff: u32) -> bool {
// }
fn get_is_quota_error(e: &serde_json::value::Value) -> bool {
trace!("get_is_quota_error");
let is_quota_error = e.get("error")
.and_then(|e| e.get("errors"))
.and_then(|e| e.get(0))