OMGeeky/twba.uploader
1
0
Fork 0
mirror of https://github.com/OMGeeky/twba.uploader.git synced 2025-12-26 16:37:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

lots of changes

Browse Source 
- implement better description & title templating and actually read description template from config
  - improve tests for this
- remove dynamic errors and create errors for everything
- remove part file after it has been uploaded
This commit is contained in:
OMGeeky
2024-04-28 01:49:41 +02:00
parent 216d690475
commit bfdc6e087d
8 changed files with 468 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -3392,9 +3392,8 @@ dependencies = [
[[package]]
name = "twba-uploader"
version = "0.2.1"
version = "0.3.0"
dependencies = [
"anyhow",
"chrono",
"futures",
"futures-util",

3
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "twba-uploader"
version = "0.2.1"
version = "0.3.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -19,7 +19,6 @@ tracing = "0.1"
tokio = { version = "1.33", features = ["rt", "rt-multi-thread", "macros"] }
thiserror = "1.0"
anyhow = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
reqwest = { version = "0.11" }

69
src/client.rs
View File

@@ -1,6 +1,8 @@
use crate::client::youtube::data::VideoData;
use crate::client::youtube::data::{create_youtube_description, create_youtube_title};
use crate::prelude::*;
use crate::CONF;
use anyhow::{anyhow, Context};
use google_youtube3::api::enums::{PlaylistStatuPrivacyStatusEnum, VideoStatuPrivacyStatusEnum};
use google_youtube3::api::Scope;
use lazy_static::lazy_static;
use std::collections::HashMap;
@@ -14,6 +16,7 @@ use twba_local_db::re_exports::sea_orm::{
ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel,
Order, QueryFilter, QueryOrder, QuerySelect,
};
use youtube::data::Location;
mod youtube;
@@ -85,8 +88,27 @@ impl UploaderClient {
let parts_folder_path = Path::new(&CONF.download_folder_path).join(video_id.to_string());
let parts = get_part_files(&parts_folder_path, part_count).await?;
dbg!(&parts);
let user = Users::find_by_id(video.user_id)
.one(&self.db)
.await?
.ok_or(UploaderError::UnknownUser(video.user_id))?;
let playlist_id = client_for_video.create_playlist(video).await?;
let tags = vec![];
let all_parts_data = VideoData {
video_tags: tags,
video_category: 22,
//TODO get from config
video_privacy: VideoStatuPrivacyStatusEnum::Private,
//TODO get from config
playlist_privacy: PlaylistStatuPrivacyStatusEnum::Private,
playlist_description: create_youtube_description(video, &user, Location::Playlist)?,
playlist_title: create_youtube_title(video, &user, Location::Playlist)?,
//The rest of the fields are filled in the loop
part_number: 0,
video_title: "".to_string(),
video_description: "".to_string(),
};
let playlist_id = client_for_video.create_playlist(&all_parts_data).await?;
self.set_playlist_id_for_video(video, playlist_id.clone())
.await?;
@@ -96,8 +118,24 @@ impl UploaderClient {
.await?
.into_active_model();
let data = VideoData {
part_number,
video_title: create_youtube_title(video, &user, Location::Video(part_number))?,
video_description: create_youtube_description(
video,
&user,
Location::Video(part_number),
)?,
..all_parts_data.clone()
};
trace!(
"uploading part {} for video: {} from path: {}",
part_number,
video.id,
part.display()
);
let upload = client_for_video
.upload_video_part(video, &part, part_number)
.upload_video_part(video, &part, part_number, data)
.await;
match upload {
Ok(uploaded_video_id) => {
@@ -167,7 +205,7 @@ impl UploaderClient {
active_video
.update(&self.db)
.await
.context("could not save video status")?;
.map_err(UploaderError::SaveVideoStatus)?;
Ok(())
}
#[tracing::instrument(skip(self, video_upload))]
@@ -187,14 +225,14 @@ impl UploaderClient {
active_video
.update(&self.db)
.await
.context("could not save video upload status")?;
.map_err(UploaderError::SaveVideoStatus)?;
Ok(())
}
fn get_client_for_video(&self, video: &VideosModel) -> Result<&youtube::YoutubeClient> {
let c = self
.youtube_client
.get(&video.user_id.to_string())
.context("could not get youtube client for video")?;
.ok_or(UploaderError::NoClient(video.user_id))?;
Ok(c)
}
}
@@ -208,21 +246,16 @@ async fn get_part_files(folder_path: &Path, count: i32) -> Result<Vec<(PathBuf,
);
let x = folder_path
.read_dir()
.context("could not read parts folder")?;
.map_err(UploaderError::ReadPartsFolder)?;
for path in x {
let path = path.context("could not read path")?;
let path = path.map_err(UploaderError::OpenPartFile)?;
let path = path.path();
let part_number = get_part_number_from_path(&path)?;
dbg!(part_number);
parts.push((path, part_number));
}
if parts.len() != count as usize {
return Err(anyhow!(
"part count does not match: expected: {}, got: {}",
count,
parts.len()
)
.into());
return Err(UploaderError::PartCountMismatch(count as usize, parts.len()).into());
}
parts.sort_by_key(|a| a.1);
Ok(parts)
@@ -237,19 +270,19 @@ fn get_part_number_from_path(path: &Path) -> Result<usize> {
if e == OsStr::new("mp4") {
let part_number = path
.file_stem()
.context("could not get file stem")?
.ok_or(UploaderError::GetNameWithoutFileExtension)?
.to_str()
.context("could not convert path to string")?
.ok_or(UploaderError::ConvertPathToString)?
.to_string();
let part_number = part_number
.parse::<usize>()
.context("could not parse path")?;
.map_err(UploaderError::ParsePartNumber)?;
return Ok(part_number + 1);
}
warn!("path has not the expected extension (.mp4): {:?}", path);
}
}
Err(anyhow!("wrong file extension").into())
Err(UploaderError::WrongFileExtension.into())
}
impl UploaderClient {

212
src/client/youtube.rs
View File

@@ -1,9 +1,6 @@
use anyhow::Context;
use chrono::{Datelike, NaiveDateTime, ParseResult, Utc};
use std::fmt::{Debug, Formatter};
use std::path::{Path, PathBuf};
use crate::client::youtube::data::VideoData;
use crate::prelude::*;
use chrono::{Datelike, NaiveDateTime, ParseResult, Utc};
use google_youtube3::api::enums::{PlaylistStatuPrivacyStatusEnum, VideoStatuPrivacyStatusEnum};
use google_youtube3::api::{
Playlist, PlaylistSnippet, PlaylistStatus, Scope, VideoSnippet, VideoStatus,
@@ -16,12 +13,15 @@ use google_youtube3::{
hyper_rustls::{HttpsConnector, HttpsConnectorBuilder},
Error as YoutubeError,
};
use std::fmt::{Debug, Formatter};
use std::path::{Path, PathBuf};
use tokio::fs;
use tracing::instrument;
use twba_local_db::entities::videos::Model;
use twba_local_db::prelude::{UsersModel, VideosModel};
mod auth;
pub(crate) mod data;
mod flow_delegate;
pub struct YoutubeClient {
@@ -31,48 +31,39 @@ pub struct YoutubeClient {
}
impl YoutubeClient {
#[instrument(skip(self, video, path))]
#[instrument(skip(self, video, path, data))]
pub(crate) async fn upload_video_part(
&self,
video: &VideosModel,
path: &Path,
part_num: usize,
data: VideoData,
) -> Result<String> {
trace!(
"uploading part {} for video: {} from path: {}",
part_num,
video.id,
path.display()
);
let title = create_youtube_title(video, TitleLocation::VideoTitle(part_num))?;
let description = format!(
"default description for video: {}",
create_youtube_title(video, TitleLocation::Descriptions)?
);
let tags = vec![];
let privacy_status = VideoStatuPrivacyStatusEnum::Private;
self.upload_youtube_video_resumable(title, description, tags, privacy_status, path)
let video_data = data;
let upload_result = self
.upload_youtube_video_resumable(video_data, path)
.await?;
fs::remove_file(path)
.await
.map_err(UploaderError::DeletePartAfterUpload)?;
Ok(upload_result)
}
async fn upload_youtube_video_resumable(
&self,
title: impl Into<String>,
description: impl Into<String>,
tags: impl Into<Vec<String>>,
privacy_status: VideoStatuPrivacyStatusEnum,
video_data: VideoData,
path: &Path,
) -> Result<String> {
let video = Video {
snippet: Some(VideoSnippet {
title: Some(title.into()),
description: Some(description.into()),
category_id: Some("20".to_string()),
tags: Some(tags.into()),
title: Some(video_data.video_title),
description: Some(video_data.video_description.into()),
category_id: Some(video_data.video_category.to_string()),
tags: Some(video_data.video_tags),
..Default::default()
}),
status: Some(VideoStatus {
privacy_status: Some(privacy_status),
privacy_status: Some(video_data.video_privacy),
public_stats_viewable: Some(true),
embeddable: Some(true),
self_declared_made_for_kids: Some(false),
@@ -81,7 +72,9 @@ impl YoutubeClient {
..Default::default()
};
let stream = fs::File::open(path).await.context("could not open file")?;
let stream = fs::File::open(path)
.await
.map_err(UploaderError::OpenPartFile)?;
let insert_call = self.client.videos().insert(video);
trace!("Starting resumable upload");
@@ -91,7 +84,11 @@ impl YoutubeClient {
trace!("Resumable upload finished");
let result_str = if upload.is_ok() { "Ok" } else { "Error" };
info!("upload request done with result: {}", result_str);
upload?.1.id.ok_or(UploaderError::Tmp2)
upload
.map_err(UploaderError::YoutubeError)?
.1
.id
.ok_or(UploaderError::NoIdReturned)
}
}
@@ -118,27 +115,25 @@ impl YoutubeClient {
.playlist_items()
.insert(playlist_item)
.doit()
.await?;
.await
.map_err(UploaderError::YoutubeError)?;
Ok(())
}
#[instrument(skip(self, video))]
pub(crate) async fn create_playlist(&self, video: &VideosModel) -> Result<String> {
pub(crate) async fn create_playlist(&self, video: &VideoData) -> Result<String> {
trace!("creating playlist for video: {:?}", video);
let title = create_youtube_title(video, TitleLocation::PlaylistTitle)?;
trace!("title: {}", title);
let description: Option<String> = None;
trace!("description: {:?}", description);
let privacy_status = PlaylistStatuPrivacyStatusEnum::Private; //TODO: Get setting per user from db
trace!("privacy: {:?}", privacy_status);
trace!("title: {}", video.playlist_title);
trace!("description: {:?}", video.playlist_description);
trace!("privacy: {:?}", video.playlist_privacy);
let playlist = Playlist {
snippet: Some(PlaylistSnippet {
title: Some(title),
description,
title: Some(video.playlist_title.clone()),
description: Some(video.playlist_description.clone()),
..Default::default()
}),
status: Some(PlaylistStatus {
privacy_status: Some(privacy_status),
privacy_status: Some(video.playlist_privacy),
}),
..Default::default()
};
@@ -152,9 +147,7 @@ impl YoutubeClient {
//test
;
Ok(playlist
.id
.context("playlist creation did not return an ID")?)
playlist.id.ok_or(UploaderError::NoIdReturned)
}
}
@@ -165,12 +158,12 @@ impl Debug for YoutubeClient {
}
impl YoutubeClient {
#[tracing::instrument]
#[tracing::instrument(skip(user), fields(user.id = user.as_ref().map(|x| x.id),user.twitch_id = user.as_ref().map(|x| &x.twitch_id)))]
pub async fn new(scopes: &Vec<Scope>, user: Option<UsersModel>) -> Result<Self> {
let hyper_client = Self::create_hyper_client();
let application_secret_path = PathBuf::from(
&shellexpand::full(&crate::CONF.google.youtube.client_secret_path)
.map_err(|e| UploaderError::ExpandPath(e.into()))?
.map_err(UploaderError::ExpandPath)?
.to_string(),
);
@@ -195,128 +188,3 @@ impl YoutubeClient {
)
}
}
enum TitleLocation {
VideoTitle(usize),
PlaylistTitle,
Descriptions,
}
fn create_youtube_title(video: &VideosModel, target: TitleLocation) -> Result<String> {
const YOUTUBE_TITLE_MAX_LENGTH: usize = 100;
let max = video.part_count as usize;
let date = parse_date(&video.created_at)
.context(format!("could not parse date: {}", &video.created_at))?;
let title = match target {
TitleLocation::VideoTitle(current) => {
let date_prefix = get_date_prefix(date.date());
let part_prefix = if current != max {
format_progress(max, current)
} else {
String::new()
};
shorten_string_if_needed(
&format!("{}{} {}", date_prefix, part_prefix, video.name),
YOUTUBE_TITLE_MAX_LENGTH,
)
}
TitleLocation::PlaylistTitle => {
let prefix = get_date_prefix(date.date());
shorten_string_if_needed(
&format!("{} {}", prefix, &video.name),
YOUTUBE_TITLE_MAX_LENGTH,
)
}
TitleLocation::Descriptions => format!("\"{}\"", video.name),
};
Ok(title)
}
fn format_progress(max: usize, current: usize) -> String {
let width = (max.checked_ilog10().unwrap_or(0) + 1) as usize;
format!("[{:0width$}/{:0width$}]", current, max, width = width)
}
const DATETIME_FORMAT: &str = "%Y-%m-%dT%H:%M:%S";
fn parse_date(date: &str) -> ParseResult<NaiveDateTime> {
Ok(chrono::DateTime::parse_from_rfc3339(date)?.naive_local())
}
fn get_date_prefix(date: chrono::NaiveDate) -> String {
format!(
"[{:0>4}-{:0>2}-{:0>2}]",
date.year(),
date.month(),
date.day()
)
}
fn shorten_string_if_needed(s: &str, target_len: usize) -> String {
const SHORTEN_CHARS: &str = "...";
if target_len < SHORTEN_CHARS.len() {
return SHORTEN_CHARS[..target_len].to_string();
}
if s.len() > target_len {
let s = &s[..target_len - SHORTEN_CHARS.len()];
let result = s.to_string() + SHORTEN_CHARS;
assert_eq!(result.len(), target_len);
result
} else {
s.to_string()
}
}
#[cfg(test)]
mod test {
use crate::client::youtube::{create_youtube_title, TitleLocation};
use twba_local_db::prelude::{Status, VideosModel};
#[test]
fn test_shorten_string() {
let test = super::shorten_string_if_needed("123456789", 50);
assert_eq!("123456789", test);
let test = super::shorten_string_if_needed("123456789", 5);
assert_eq!("12...", test);
let test = super::shorten_string_if_needed("123456789", 3);
assert_eq!("...", test);
let test = super::shorten_string_if_needed("123456789", 2);
assert_eq!("..", test);
let test = super::shorten_string_if_needed("123456789", 0);
assert_eq!("", test);
}
#[test]
fn test_create_youtube_title() {
let mut x = VideosModel {
part_count: 4,
name: "wow".to_string(),
created_at: "2023-10-09T19:33:59".to_string(),
//the rest is just dummy data
id: 3,
status: Status::Uploading,
user_id: 0,
twitch_id: String::new(),
twitch_preview_image_url: None,
twitch_download_url: None,
duration: 0,
youtube_id: None,
youtube_playlist_name: String::new(),
youtube_preview_image_url: None,
youtube_playlist_id: None,
youtube_playlist_created_at: None,
fail_count: 0,
fail_reason: None,
};
let description = create_youtube_title(&x, TitleLocation::Descriptions).unwrap();
assert_eq!("\"wow\"", description);
let playlist = create_youtube_title(&x, TitleLocation::PlaylistTitle).unwrap();
assert_eq!("[2023-10-09] wow", playlist);
let video = create_youtube_title(&x, TitleLocation::VideoTitle(2)).unwrap();
assert_eq!("[2023-10-09][2/4] wow", video);
x.part_count = 14;
let video = create_youtube_title(&x, TitleLocation::VideoTitle(2)).unwrap();
assert_eq!("[2023-10-09][02/14] wow", video);
}
}

24
src/client/youtube/auth.rs
View File

@@ -1,6 +1,6 @@
use crate::client::youtube::flow_delegate::CustomFlowDelegate;
use crate::errors::{AuthError, PersistentPathError};
use crate::prelude::*;
use anyhow::{anyhow, Context};
use google_youtube3::api::Scope;
use google_youtube3::{hyper::client::HttpConnector, hyper_rustls::HttpsConnector, oauth2};
use std::collections::HashMap;
@@ -9,6 +9,8 @@ use std::path::{Path, PathBuf};
use tokio::fs;
use tracing::instrument;
use yup_oauth2::authenticator::Authenticator;
type Result<T> = std::result::Result<T, AuthError>;
#[instrument]
pub(super) async fn get_auth<USER: EasyString>(
application_secret_path: &impl EasyPath,
@@ -25,7 +27,7 @@ pub(super) async fn get_auth<USER: EasyString>(
let app_secret = oauth2::read_application_secret(application_secret_path)
.await
.context("could not read application secret from path")?;
.map_err(AuthError::ReadApplicationSecret)?;
let persistent_path =
get_and_validate_persistent_path(&crate::CONF.google.path_auth_cache, user.clone()).await?;
@@ -44,13 +46,13 @@ pub(super) async fn get_auth<USER: EasyString>(
.force_account_selection(true)
.build()
.await
.context("error creating authenticator")?;
.map_err(AuthError::CreateAuth)?;
trace!("got authenticator, requesting scopes");
let access_token = auth
.token(scopes)
.await
.context("could not get access to the requested scopes")?;
.map_err(|e| AuthError::GetAccessToken(e.into()))?;
trace!("got scope access: {:?}", access_token);
Ok(auth)
}
@@ -73,7 +75,7 @@ async fn get_and_validate_persistent_path<TEMPLATE: EasyString, USER: EasyString
let persistent_path_parent_folder = persistent_path
.parent()
.context("could not get parent folder")?;
.ok_or(PersistentPathError::GetParentFolder)?;
if !persistent_path_parent_folder.exists() {
debug!(
"persistent path parent folder does not exist, creating it: {}",
@@ -81,17 +83,15 @@ async fn get_and_validate_persistent_path<TEMPLATE: EasyString, USER: EasyString
);
fs::create_dir_all(persistent_path_parent_folder)
.await
.context("could not create dirs")?;
.map_err(PersistentPathError::CreateDirs)?;
} else if !persistent_path_parent_folder.is_dir() {
error!(
"persistent path parent folder is not a dir: {}",
persistent_path_parent_folder.display()
);
return Err(anyhow!(
"persistent path parent folder is not a dir: {}",
persistent_path_parent_folder.display()
)
.into());
return Err(
PersistentPathError::PathNotDir(persistent_path_parent_folder.to_path_buf()).into(),
);
}
Ok(persistent_path.to_path_buf())
}
@@ -106,6 +106,6 @@ fn get_persistent_path<TEMPLATE: EasyString, USER: EasyString>(
};
let vars: HashMap<String, String> = HashMap::from([("user".to_string(), user)]);
let persistent_path = strfmt::strfmt(&persistent_path_template.into(), &vars)
.context("could not replace user in persistent path")?;
.map_err(PersistentPathError::ReplaceUser)?;
Ok(persistent_path)
}

299
src/client/youtube/data.rs Normal file
View File

@@ -0,0 +1,299 @@
use crate::client::youtube::data::substitutions::*;
use crate::prelude::*;
use crate::CONF;
use chrono::{Datelike, NaiveDateTime, ParseResult};
use google_youtube3::api::enums::{PlaylistStatuPrivacyStatusEnum, VideoStatuPrivacyStatusEnum};
use std::fmt::Debug;
use twba_local_db::prelude::{UsersModel, VideosModel};
/// The maximum length of a YouTube title that is allowed
///
/// This is a constant because it is a hard limit set by YouTube
const YOUTUBE_TITLE_MAX_LENGTH: usize = 100;
pub mod substitutions {
pub const ORIGINAL_TITLE: &str = "$$original_title$$";
pub const ORIGINAL_DESCRIPTION: &str = "$$original_description$$";
pub const UPLOAD_DATE: &str = "$$upload_date$$";
pub const UPLOAD_DATE_SHORT: &str = "$$upload_date_short$$";
pub const TWITCH_URL: &str = "$$twitch_url$$";
pub const TWITCH_CHANNEL_NAME: &str = "$$twitch_channel_name$$";
pub const TWITCH_CHANNEL_URL: &str = "$$twitch_channel_url$$";
pub const PART_COUNT: &str = "$$part_count$$";
pub const PART_IDENT: &str = "$$part_ident$$";
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Location {
Video(usize),
Playlist,
Other,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct VideoData {
pub part_number: usize,
pub video_title: String,
pub video_description: String,
pub video_tags: Vec<String>,
pub video_category: u32,
pub video_privacy: VideoStatuPrivacyStatusEnum,
pub playlist_title: String,
pub playlist_description: String,
pub playlist_privacy: PlaylistStatuPrivacyStatusEnum,
}
pub struct Templates {
pub video_title: String,
pub video_description: String,
pub playlist_title: String,
pub playlist_description: String,
}
impl Default for Templates {
fn default() -> Self {
Self {
video_title: format!("[{}]{} {}", UPLOAD_DATE_SHORT, PART_IDENT, ORIGINAL_TITLE),
video_description: format!(
"default description for video: {} from {}\n\nOriginal stream here: \n{}\n\nWatch {} live at: {}",
ORIGINAL_TITLE, UPLOAD_DATE, TWITCH_URL, TWITCH_CHANNEL_NAME, TWITCH_CHANNEL_URL
),
playlist_title: format!("[{}] {}", UPLOAD_DATE_SHORT, ORIGINAL_TITLE),
playlist_description: format!(
"default description for video: {} from {}\n\nOriginal stream here: \n{}\n\nWatch {} live at: {}",
ORIGINAL_TITLE, UPLOAD_DATE, TWITCH_URL, TWITCH_CHANNEL_NAME, TWITCH_CHANNEL_URL
),
}
}
}
pub(crate) fn create_youtube_description(
video: &VideosModel,
user: &UsersModel,
target: Location,
) -> Result<String> {
let s = get_description_template(target);
let description = substitute(s, video, user, target)?;
Ok(description)
}
pub(crate) fn create_youtube_title(
video: &VideosModel,
user: &UsersModel,
target: Location,
) -> Result<String> {
let title_template = get_title_template(target);
let title = substitute(title_template, video, user, target)?;
let max_len = match target {
Location::Video(_) => Some(YOUTUBE_TITLE_MAX_LENGTH),
Location::Playlist => Some(YOUTUBE_TITLE_MAX_LENGTH),
Location::Other => None,
};
shorten_string_if_needed(&title, max_len);
Ok(title)
}
fn get_title_template(target: Location) -> String {
let templates = Templates::default();
match target {
Location::Video(_) => templates.video_title,
Location::Playlist => templates.playlist_title,
Location::Other => format!("\"{}\"", ORIGINAL_TITLE),
}
}
fn get_description_template(target: Location) -> String {
let configured = &CONF.google.youtube.default_description_template;
if !configured.is_empty() {
return configured.to_string();
}
let templates = Templates::default();
match target {
Location::Video(_) => templates.video_description,
Location::Playlist => templates.playlist_description,
Location::Other => templates.video_description,
}
}
fn substitute(
input: String,
video: &VideosModel,
user: &UsersModel,
target: Location,
) -> Result<String> {
let max = video.part_count as usize;
let s = substitute_common(input, video, user, max)?;
let title = match target {
Location::Video(current) => substitute_part_ident(&s, current, max),
_ => s,
};
Ok(title)
}
fn substitute_part_ident(input: &str, current: usize, max: usize) -> String {
let part_prefix = if max > 1 {
format_progress(max, current)
} else {
String::new()
};
input.replace(PART_IDENT, &part_prefix)
}
fn substitute_common(
input: String,
video: &VideosModel,
user: &UsersModel,
max: usize,
) -> Result<String> {
let date = parse_date(&video.created_at).map_err(UploaderError::ParseDate)?;
let date_prefix = get_date_prefix(date.date());
Ok(input
.replace(ORIGINAL_TITLE, &video.name)
.replace(ORIGINAL_DESCRIPTION, "")
.replace(UPLOAD_DATE, &date.to_string())
.replace(UPLOAD_DATE_SHORT, &date_prefix)
.replace(
TWITCH_URL,
video.twitch_download_url.as_ref().unwrap_or(&String::new()),
)
.replace(TWITCH_CHANNEL_NAME, &user.twitch_name)
.replace(
TWITCH_CHANNEL_URL,
&format!("https://twitch.tv/users/{}", &user.twitch_id),
)
.replace(PART_COUNT, &max.to_string()))
}
fn shorten_string_if_needed(s: &str, target_len: Option<usize>) -> String {
const SHORTEN_CHARS: &str = "...";
if target_len.is_none() {
return s.to_string();
}
let target_len = target_len.unwrap();
if target_len < SHORTEN_CHARS.len() {
return SHORTEN_CHARS[..target_len].to_string();
}
if s.len() > target_len {
let s = &s[..target_len - SHORTEN_CHARS.len()];
let result = s.to_string() + SHORTEN_CHARS;
assert_eq!(result.len(), target_len);
result
} else {
s.to_string()
}
}
fn get_date_prefix(date: chrono::NaiveDate) -> String {
format!(
"{:0>4}-{:0>2}-{:0>2}",
date.year(),
date.month(),
date.day()
)
}
fn format_progress(max: usize, current: usize) -> String {
let width = (max.checked_ilog10().unwrap_or(0) + 1) as usize;
format!("[{:0width$}/{:0width$}]", current, max, width = width)
}
fn parse_date(date: &str) -> ParseResult<NaiveDateTime> {
Ok(chrono::DateTime::parse_from_rfc3339(date)?.naive_local())
}
#[cfg(test)]
mod test {
use crate::client::youtube::data::create_youtube_title;
use crate::client::youtube::data::Location;
use twba_local_db::prelude::{Status, UsersModel, VideosModel};
#[test]
fn test_shorten_string() {
let test = super::shorten_string_if_needed("123456789", Some(50));
assert_eq!("123456789", test);
let test = super::shorten_string_if_needed("123456789", Some(5));
assert_eq!("12...", test);
let test = super::shorten_string_if_needed("123456789", Some(3));
assert_eq!("...", test);
let test = super::shorten_string_if_needed("123456789", Some(2));
assert_eq!("..", test);
let test = super::shorten_string_if_needed("123456789", Some(0));
assert_eq!("", test);
let test = super::shorten_string_if_needed("123456789", None);
assert_eq!("123456789", test);
}
#[test]
fn test_create_youtube_title_other() {
let (mut x, user) = get_test_sample_data();
let description = create_youtube_title(&x, &user, Location::Other).unwrap();
assert_eq!("\"wow\"", description);
}
#[test]
fn test_create_youtube_title_playlist() {
let (mut x, user) = get_test_sample_data();
let playlist = create_youtube_title(&x, &user, Location::Playlist).unwrap();
assert_eq!("[2023-10-09] wow", playlist);
}
#[test]
fn test_create_youtube_title_video_1() {
let (mut x, user) = get_test_sample_data();
let video = create_youtube_title(&x, &user, Location::Video(1)).unwrap();
assert_eq!("[2023-10-09][1/4] wow", video);
}
#[test]
fn test_create_youtube_title_video_2() {
let (mut x, user) = get_test_sample_data();
let video = create_youtube_title(&x, &user, Location::Video(2)).unwrap();
assert_eq!("[2023-10-09][2/4] wow", video);
}
#[test]
fn test_create_youtube_title_video_3() {
let (mut x, user) = get_test_sample_data();
let video = create_youtube_title(&x, &user, Location::Video(3)).unwrap();
assert_eq!("[2023-10-09][3/4] wow", video);
}
#[test]
fn test_create_youtube_title_video_4() {
let (mut x, user) = get_test_sample_data();
let video = create_youtube_title(&x, &user, Location::Video(4)).unwrap();
assert_eq!("[2023-10-09][4/4] wow", video);
}
#[test]
fn test_create_youtube_title_video_multi_digit_part_count() {
let (mut x, user) = get_test_sample_data();
x.part_count = 14;
let video = create_youtube_title(&x, &user, Location::Video(2)).unwrap();
assert_eq!("[2023-10-09][02/14] wow", video);
}
fn get_test_sample_data() -> (VideosModel, UsersModel) {
let mut x = VideosModel {
part_count: 4,
name: "wow".to_string(),
created_at: "2023-10-09T19:33:59+00:00".to_string(),
//the rest is just dummy data
id: 3,
status: Status::Uploading,
user_id: 0,
twitch_id: String::new(),
twitch_preview_image_url: None,
twitch_download_url: None,
duration: 0,
youtube_id: None,
youtube_playlist_name: String::new(),
youtube_preview_image_url: None,
youtube_playlist_id: None,
youtube_playlist_created_at: None,
fail_count: 0,
fail_reason: None,
};
let user = UsersModel {
id: 0,
twitch_id: "".to_string(),
twitch_name: "".to_string(),
twitch_profile_image_url: None,
youtube_id: "".to_string(),
youtube_name: "".to_string(),
youtube_profile_image_url: None,
youtube_target_duration: 0,
youtube_max_duration: 0,
active: false,
};
(x, user)
}
}

6
src/client/youtube/flow_delegate.rs
View File

@@ -1,6 +1,5 @@
use crate::errors::AuthError;
use crate::prelude::*;
use anyhow::anyhow;
use twba_backup_config::Conf;
use std::{
fmt::{Debug, Formatter},
future::Future,
@@ -8,6 +7,7 @@ use std::{
pin::Pin,
};
use tracing::instrument;
use twba_backup_config::Conf;
use yup_oauth2::authenticator_delegate::InstalledFlowDelegate;
pub struct CustomFlowDelegate<USER: EasyString> {
@@ -91,7 +91,7 @@ async fn get_auth_code() -> Result<String> {
if e.kind() != std::io::ErrorKind::NotFound {
println!("Error removing file: {}", e);
error!("Error removing file: {}", e);
return Err(anyhow!("Error removing file: {}", e).into());
return Err(AuthError::RemoveAuthCodeFile(e).into());
}
}
let message = format!("Waiting for auth code in file: {}", path.display());

74
src/errors.rs
View File

@@ -1,24 +1,72 @@
use shellexpand::LookupError;
use std::env::VarError;
use std::path::PathBuf;
use strfmt::FmtError;
#[derive(Debug, thiserror::Error)]
pub enum UploaderError {
#[error("Path could not be expanded")]
ExpandPath(#[source] anyhow::Error),
ExpandPath(#[source] LookupError<VarError>),
#[error("Could not load config")]
LoadConfig(#[source] anyhow::Error),
#[error("Got an auth error: {0}")]
AuthError(#[from] AuthError),
#[error("Some error with the database: {0:?}")]
OpenDatabase(#[from] twba_local_db::re_exports::sea_orm::DbErr),
Database(#[from] twba_local_db::re_exports::sea_orm::DbErr),
#[error("Error with some Youtube operation: {0} ")]
YoutubeError(#[source] google_youtube3::Error),
#[error("Temporary error. Remove for production, {0}")]
//TODO: Remove this error
Tmp1(#[from] anyhow::Error),
#[error("Temporary error. Remove for production, {0}")]
//TODO: Remove this error
Tmp3(#[from] google_youtube3::Error),
#[error("Temporary error. Remove for production")]
//TODO: Remove this error
Tmp2,
#[error("Could not find user: {0}")]
UnknownUser(i32),
#[error("Could not find client for user: {0}")]
NoClient(i32),
#[error("Could not read part file: {0}")]
OpenPartFile(#[source] std::io::Error),
#[error("Could not read parts folder: {0}")]
ReadPartsFolder(#[source] std::io::Error),
#[error("Could not delete part file after uploading: {0}")]
DeletePartAfterUpload(#[source] std::io::Error),
#[error("wrong file extension")]
WrongFileExtension,
#[error("could not get file stem")]
GetNameWithoutFileExtension,
#[error("could not convert path to string")]
ConvertPathToString,
#[error("could not parse part number from path: {0}")]
ParsePartNumber(#[source] std::num::ParseIntError),
#[error("could not save video status")]
SaveVideoStatus(#[source] twba_local_db::re_exports::sea_orm::DbErr),
#[error("could not parse date: {0}")]
ParseDate(#[source] chrono::ParseError),
#[error("part count does not match: expected: {0}, got: {1}")]
PartCountMismatch(usize, usize),
#[error("no id returned from youtube")]
NoIdReturned,
}
#[derive(Debug, thiserror::Error)]
pub enum AuthError {
#[error("could not read application secret from path: {0}")]
ReadApplicationSecret(#[source] std::io::Error),
#[error("could not create auth")]
CreateAuth(#[source] std::io::Error),
#[error("could not get access to the requested scopes")]
GetAccessToken(#[source] Box<dyn std::error::Error>),
#[error("could not get and validate persistent path: {0}")]
PersistentPathError(#[from] PersistentPathError),
#[error("could not remove existing auth code file: {0}")]
RemoveAuthCodeFile(#[source] std::io::Error),
}
#[derive(Debug, thiserror::Error)]
pub enum PersistentPathError {
#[error("persistent path parent folder is not a dir: {0}")]
PathNotDir(PathBuf),
#[error("could not replace user in persistent path")]
ReplaceUser(#[source] FmtError),
#[error("could not get parent folder")]
GetParentFolder,
#[error("could not create dirs")]
CreateDirs(#[source] std::io::Error),
}