From 207e2fadf4b4a12281f136bb26444d843f9cce58 Mon Sep 17 00:00:00 2001 From: OMGeeky Date: Wed, 6 Sep 2023 18:35:04 +0200 Subject: [PATCH] misc --- Cargo.toml | 3 ++ logger.yaml | 12 ++++---- src/lib.rs | 19 +++++++----- src/main.rs | 76 ++++++++++++++++++++++++++++++++++++---------- src/prelude.rs | 7 +---- tests/lib_tests.rs | 32 ++++++++++++++++++- 6 files changed, 112 insertions(+), 37 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 334f2a2..48457db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ log = "0.4" tracing = "0.1" data-test = "0.1.1" +tracing-appender = "0.2" +tracing-subscriber = "0.3" + [patch.crates-io] # patch the yup-oauth2 version with a custom for to support forcing the user to choose an account. # this can be removed as soon as https://github.com/dermesser/yup-oauth2/ has its next release and diff --git a/logger.yaml b/logger.yaml index 559f898..067311f 100644 --- a/logger.yaml +++ b/logger.yaml @@ -11,7 +11,7 @@ appenders: trace_file: kind: rolling_file - path: "/var/tmp/twba/logs/downloader.trace.log" + path: "/downloader/logs/downloader.trace.log" filters: - kind: threshold level: trace @@ -24,12 +24,12 @@ appenders: limit: 1 gb roller: kind: fixed_window - pattern: "/var/tmp/twba/logs/archive/downloader.trace.{}.log" + pattern: "/downloader/logs/archive/downloader.trace.{}.log" count: 5 info_file: kind: rolling_file - path: "/var/tmp/twba/logs/downloader.info.log" + path: "/downloader/logs/downloader.info.log" filters: - kind: threshold level: info @@ -42,12 +42,12 @@ appenders: limit: 100mb roller: kind: fixed_window - pattern: "/var/tmp/twba/logs/archive/downloader.info.{}.log" + pattern: "/downloader/logs/archive/downloader.info.{}.log" count: 5 debug_file: kind: rolling_file - path: "/var/tmp/twba/logs/downloader.debug.log" + path: "/downloader/logs/downloader.debug.log" filters: - kind: threshold level: debug @@ -60,7 +60,7 @@ appenders: limit: 1gb roller: kind: fixed_window - pattern: "/var/tmp/twba/logs/archive/downloader.debug.{}.log" + pattern: "/downloader/logs/archive/downloader.debug.{}.log" count: 5 root: diff --git a/src/lib.rs b/src/lib.rs index d7964db..b46eb07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -148,7 +148,7 @@ async fn get_youtube_clients( let mut result = HashMap::new(); let config = load_config(); let streamers = get_watched_streamers(db_client).await?; - + info!("Getting youtube clients for {:?}", streamers); for streamer in streamers { trace!("Creating youtube client"); @@ -169,8 +169,10 @@ async fn get_youtube_clients( ) .await .map_err(|e| anyhow!("error creating the youtube client: {}", e))?; + info!("Got client for user: {}", user); result.insert(user, youtube_client); } + info!("Got youtube clients"); Ok(result) } @@ -438,7 +440,7 @@ pub async fn split_video_into_parts( let filepath = path.canonicalize()?; let parent_dir = path.parent().unwrap().canonicalize(); if parent_dir.is_err() { - warn!("Could not canonicalize parent dir"); + warn!("Could not canonicalize parent dir: {:?}", path); } let parent_dir = parent_dir.expect("Could not canonicalize parent dir"); @@ -533,9 +535,10 @@ pub async fn split_video_into_parts( .expect("to_str on path did not work!"); let last_path = clean(&last_path); let last_path = last_path.to_str().expect("to_str on path did not work!"); + //create a file to tell ffmpeg what files to join/concat tokio::fs::write( join_txt_path.clone(), - format!("file '{}'\nfile '{}'", last_path, second_last_path_str,), + format!("file '{}'\nfile '{}'", second_last_path_str, last_path), ) .await?; @@ -885,16 +888,16 @@ mod tests { let (total_time, parts) = extract_track_info_from_playlist(sample_playlist_content) .expect("failed to extract track info from playlist"); - assert_eq!(total_time, 18002.0 as f64); + assert_eq!(total_time, 18002.0f64); assert_eq!(parts.len(), 2); assert_eq!( parts[0], - ("1740252892.mp4_000.mp4".to_string(), 18001.720898 as f64) + ("1740252892.mp4_000.mp4".to_string(), 18001.720898f64) ); assert_eq!( parts[1], - ("1740252892.mp4_001.mp4".to_string(), 14633.040755 as f64) + ("1740252892.mp4_001.mp4".to_string(), 14633.040755f64) ); } #[tokio::test] @@ -920,8 +923,8 @@ mod tests { ); assert_eq!(parts[0], Path::join(parent_dir, "1740252892.mp4_000.mp4")); assert_eq!(parts[1], Path::join(parent_dir, "1740252892.mp4_001.mp4")); - assert_eq!(second_last_time, 18001.720898 as f64); - assert_eq!(last_time, 14633.040755 as f64); + assert_eq!(second_last_time, 18001.720898f64); + assert_eq!(last_time, 14633.040755f64); } } diff --git a/src/main.rs b/src/main.rs index 0370ed5..c4e9a6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,10 +5,10 @@ use std::fmt::Debug; use std::path::Path; use anyhow::{anyhow, Result}; +use downloader::prelude::*; use google_bigquery_v2::prelude::*; use google_youtube::scopes; use google_youtube::YoutubeClient; -use downloader::prelude::*; use log4rs::append::console::ConsoleAppender; use log4rs::append::rolling_file::policy::compound::roll; use log4rs::append::rolling_file::policy::compound::roll::fixed_window::FixedWindowRoller; @@ -18,8 +18,11 @@ use log4rs::append::rolling_file::{RollingFileAppender, RollingFileAppenderBuild use log4rs::config::{Appender, Root}; use log4rs::encode::pattern::PatternEncoder; use nameof::name_of; -use simplelog::*; use tokio::fs::File; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_appender::rolling; +use tracing_appender::rolling::Rotation; +use tracing_subscriber::prelude::*; use twitch_data::{ convert_twitch_video_to_twitch_data_video, get_client, TwitchClient, Video, VideoQuality, }; @@ -37,12 +40,46 @@ const DATASET_ID: &str = "backup_data"; #[tokio::main] async fn main() -> Result<(), Box> { - initialize_logger2().await; + // initialize_logger2().await; + let _guards = initialize_logger3().await; info!("Hello, world!"); start_backup().await?; // sample().await?; Ok(()) } +async fn initialize_logger3() -> Result<(WorkerGuard, WorkerGuard, WorkerGuard), Box> { + let (info_daily, guard_info_daily) = + tracing_appender::non_blocking(rolling::daily("/downloader/logs", "info.log")); + let (debug_daily, guard_debug_daily) = + tracing_appender::non_blocking(rolling::daily("/downloader/logs", "debug.log")); + let (trace_daily, guard_trace_daily) = + tracing_appender::non_blocking(rolling::daily("/downloader/logs", "trace.log")); + + let info_layer = tracing_subscriber::fmt::layer() + .with_writer(info_daily) + .with_filter(tracing_subscriber::filter::LevelFilter::INFO); + + let debug_layer = tracing_subscriber::fmt::layer() + .with_writer(debug_daily) + .with_filter(tracing_subscriber::filter::LevelFilter::DEBUG); + + let trace_layer = tracing_subscriber::fmt::layer() + .with_writer(trace_daily) + .with_filter(tracing_subscriber::filter::LevelFilter::TRACE); + + let stdout_layer = + tracing_subscriber::fmt::layer().with_filter(tracing_subscriber::filter::LevelFilter::INFO); + + let subscriber = tracing_subscriber::registry() + .with(info_layer) + .with(debug_layer) + .with(trace_layer) + .with(stdout_layer); + + tracing::subscriber::set_global_default(subscriber).expect("Failed to set global subscriber"); + print_log_start_msg(); + Ok((guard_info_daily, guard_debug_daily, guard_trace_daily)) +} async fn initialize_logger2() -> Result<(), Box> { // // example: @@ -84,14 +121,19 @@ async fn initialize_logger2() -> Result<(), Box> { ); log4rs::init_file(path, Default::default()) .expect("Failed to initialize the logger from the file"); + + info!("=================================================================================="); + + Ok(()) +} + +fn print_log_start_msg() { info!("=================================================================================="); info!( "Start of new log on {}", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S") ); info!("=================================================================================="); - - Ok(()) } fn gb_to_bytes(gb: f32) -> u64 { @@ -103,25 +145,25 @@ async fn initialize_logger() -> Result<(), Box> { tokio::fs::create_dir_all(log_folder).await?; let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H-%M-%S").to_string(); - CombinedLogger::init(vec![ + simplelog::CombinedLogger::init(vec![ // SimpleLogger::new(LevelFilter::Info, Config::default()), - TermLogger::new( + simplelog::TermLogger::new( LevelFilter::Info, - Config::default(), - TerminalMode::Mixed, - ColorChoice::Auto, + simplelog::Config::default(), + simplelog::TerminalMode::Mixed, + simplelog::ColorChoice::Auto, ), - WriteLogger::new( + simplelog::WriteLogger::new( LevelFilter::Info, - Config::default(), + simplelog::Config::default(), File::create(format!("{}downloader_{}.log", log_folder, timestamp)) .await? .into_std() .await, ), - WriteLogger::new( + simplelog::WriteLogger::new( LevelFilter::Trace, - Config::default(), + simplelog::Config::default(), File::create(format!("{}trace_{}.log", log_folder, timestamp)) .await? .into_std() @@ -194,7 +236,8 @@ async fn sample_bigquery<'a>(client: &'a BigqueryClient) -> Result<(), Box(client: &'a BigqueryClient) -> Result<(), Box (PathBuf, PathBuf) { std::fs::copy(video_source, &video_path).unwrap(); (tmp_folder_path.to_path_buf(), video_path) } + +#[tokio::test] +async fn download_video_with_multi_parts() { + init_console_logging(LevelFilter::Debug); + //sample video: https://www.twitch.tv/videos/1592654401 + //total length 01:10:13 (HH:MM:SS) + let x = twitch_data::get_client() + .await + .expect("could not get client"); + let path = temp_dir(); + info!("downloading test video to folder: {}", path.display()); + let video_path = x + .download_video("1592654401", "160p30", &path) + .await + .expect("could not download video"); + info!("downloaded video: {:?}", video_path); + info!("splitting test video"); + let paths = downloader::split_video_into_parts( + video_path, + Duration::minutes(20), + Duration::minutes(35), + ) + .await + .expect("could not split video"); + + info!("checking results"); + assert_eq!(paths.len(), 3); + info!(?paths); +}