From d7e99115f3a0b6576251957aa02d29f911c53848 Mon Sep 17 00:00:00 2001 From: OMGeeky Date: Tue, 24 Jan 2023 18:18:05 +0100 Subject: [PATCH] backup --- src/errors.rs | 40 +++++++++++++ src/youtube.rs | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 src/errors.rs create mode 100644 src/youtube.rs diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..25bb98f --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,40 @@ +use std::fmt; + +#[derive(Debug, Clone)] +pub struct BackoffError { + message: String, +} + +impl BackoffError { + pub fn new>(message: S) -> BackoffError { + let message = message.into(); + BackoffError { message } + } +} + + +// Generation of an error is completely separate from how it is displayed. +// There's no need to be concerned about cluttering complex logic with the display style. +// +// Note that we don't store any extra info about the errors. This means we can't state +// which string failed to parse without modifying our types to carry that information. +impl fmt::Display for BackoffError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}: {}", stringify!(BackoffError), self.message) + } +} + +impl From<&str> for BackoffError { + fn from(s: &str) -> BackoffError { + BackoffError { + message: s.to_string(), + } + } +} + +impl std::error::Error for BackoffError { + fn description(&self) -> &str { + self.message.as_str() + } + +} \ No newline at end of file diff --git a/src/youtube.rs b/src/youtube.rs new file mode 100644 index 0000000..b5adeac --- /dev/null +++ b/src/youtube.rs @@ -0,0 +1,153 @@ +use std::error::Error; +use std::future::Future; +use std::path::{Path, PathBuf}; + +use google_youtube3::api::Video; +use google_youtube3::Error::BadRequest; +use google_youtube3::hyper::{Body, Response}; +use google_youtube3::hyper::client::HttpConnector; +use google_youtube3::hyper_rustls::HttpsConnector; +use google_youtube3::YouTube; + +const YOUTUBE_DEFAULT_BACKOFF_TIME_S: u64 = 2; +const YOUTUBE_MAX_BACKOFF_TIME_S: u64 = 3600; + +struct UploadParameters { + video: Video, + path: PathBuf, + mime_type: mime::Mime +} + +//TODO: implement backoff for other youtube calls + +async fn generic_check_backoff_youtube< + T, + // Fut: Future, T)>, Box>> , + Fut: Future, T), google_youtube3::Error>>, + Para> +( + client: &YouTube>, + para: &Para, + function: impl Fn(&YouTube>, &Para) -> Fut +) + -> Result, T)>, Box> +// where Fut: Future, T)>> +{ + let mut backoff = 0; + let mut res: google_youtube3::Result<(Response, T)>; + 'try_upload: loop { + // let stream = tokio::fs::File::open(&path).await?; + // let stream = stream.into_std().await; + // println!("Uploading video ({}): {:?}", backoff, path.as_ref().to_str()); + // + // res = client.videos().insert(video.clone()).upload(stream, mime_type.clone()).await; + res = function(&client, para).await; + match res { + Ok(_) => break 'try_upload, + Err(e) => { + println!("Error: {}", e); + if let BadRequest(e1) = &e { + let is_quota_error = get_is_quota_error(&e1); + backoff += 1; + + println!("is_quota_error: {}", is_quota_error); + if is_quota_error { + let backoff_time = YOUTUBE_DEFAULT_BACKOFF_TIME_S.pow(backoff); + println!("backoff_time: {}", backoff_time); + if backoff_time > YOUTUBE_MAX_BACKOFF_TIME_S { + return Err(e.into()); + } + //TODO: test this backoff + tokio::time::sleep(std::time::Duration::from_millis(backoff_time * 1000)).await; + } else { + return Err(e.into()); + } + } else { + return Err(e.into()); + } + } + } + } + + let res: google_youtube3::Result<(Response, T)> = res; + Ok(res) +} + +pub async fn check_backoff_youtube_upload(client: &YouTube>, + video: Video, + path: impl AsRef, + mime_type: mime::Mime) + -> Result, Video)>, Box> +{ + let params = UploadParameters { + video: video.clone(), + path: path.as_ref().into(), + mime_type: mime_type.clone() + }; + + async fn function(client: &YouTube>, para: &UploadParameters) + -> Result<(Response, Video), google_youtube3::Error> { + // let para = para.get_parameters(); + // let stream = tokio::fs::File::open(¶.path).await?; + // let stream = stream.into_std().await; + let stream = std::fs::File::open(¶.path)?; + // println!("Uploading video ({}): {:?}", backoff, path.as_ref().to_str()); + client.videos().insert(para.video.clone()).upload(stream, para.mime_type.clone()).await + } + let res = generic_check_backoff_youtube(client, ¶ms, function).await??; + + + // let mut backoff = 0; + // let mut res: google_youtube3::Result<(Response, Video)>; + // 'try_upload: loop { + // let stream = tokio::fs::File::open(&path).await?; + // let stream = stream.into_std().await; + // println!("Uploading video ({}): {:?}", backoff, path.as_ref().to_str()); + // res = client.videos().insert(video.clone()).upload(stream, mime_type.clone()).await; + // match res { + // Ok(_) => break 'try_upload, + // Err(e) => { + // println!("Error: {}", e); + // if let BadRequest(e1) = &e { + // let is_quota_error = get_is_quota_error(&e1); + // backoff += 1; + // + // println!("is_quota_error: {}", is_quota_error); + // if is_quota_error { + // let backoff_time = YOUTUBE_DEFAULT_BACKOFF_TIME_S.pow(backoff); + // println!("backoff_time: {}", backoff_time); + // if backoff_time > YOUTUBE_MAX_BACKOFF_TIME_S { + // return Err(e.into()); + // } + // //TODO: test this backoff + // tokio::time::sleep(std::time::Duration::from_millis(backoff_time * 1000)).await; + // } else { + // return Err(e.into()); + // } + // } else { + // return Err(e.into()); + // } + // } + // } + // } + // + + let res: google_youtube3::Result<(Response, Video)> = Ok(res); + Ok(res) +} + +fn get_is_quota_error(e: &serde_json::value::Value) -> bool { + let is_quota_error = e.get("error") + .and_then(|e| e.get("errors")) + .and_then(|e| e.get(0)) + .and_then(|e| e.get("reason")) + .and_then(|e| e.as_str()) + .and_then(|e| + if e == "quotaExceeded" { + Some(()) + } else { + None + } + ).is_some(); + is_quota_error +}