diff --git a/.dockerignore b/.dockerignore index 64005f9..4c46284 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ # ingore everything * # include the build -!build/downloader \ No newline at end of file +!build/downloader +!logger.yaml \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index d41d313..6998330 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + [[package]] name = "aliri_braid" version = "0.3.1" @@ -359,6 +368,7 @@ version = "0.1.4" dependencies = [ "chrono", "downloader_config", + "env_logger", "google-bigquery2", "google_bigquery", "google_youtube", @@ -401,6 +411,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +dependencies = [ + "humantime", + "is-terminal", + "log 0.4.17", + "regex", + "termcolor", +] + [[package]] name = "errno" version = "0.3.0" @@ -952,6 +975,18 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1402,6 +1437,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "reqwest" version = "0.11.16" @@ -2377,6 +2429,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-targets" version = "0.42.2" diff --git a/Cargo.toml b/Cargo.toml index da8c7ad..716d6a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ log4rs = { version = "1.2.0" , features = ["compound_policy", "default", "size_t path-clean = "1.0.1" log-panics = { version = "2", features = ["with-backtrace"]} +env_logger = "0.10.0" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 746fc16..c2f87e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,8 @@ RUN apk add libgcc # add ffmpeg to the container (needed for video splitting) RUN apk add ffmpeg +COPY ./logger.yaml ./logger.yaml + # copy the binary from the build folder # this binary should be build with the # target x86_64-unknown-linux-musl to be able to run diff --git a/logger.yaml b/logger.yaml index 7577310..e7f7463 100644 --- a/logger.yaml +++ b/logger.yaml @@ -3,25 +3,70 @@ refresh_rate: 30 seconds appenders: stdout: kind: console + filters: + - kind: threshold + level: info + encoder: + pattern: "[{d(%Y-%m-%d %H:%M:%S)}] [{h({l:5})}] {M} - {m}{n}" - requests: - kind: file - path: "/tmp/twba/logs/downloader.log" + trace_file: + kind: rolling_file + path: "/tmp/twba/logs/downloader.trace.log" + filters: + - kind: threshold + level: trace encoder: pattern: "[{d:35}] [{h({l:5})}] {M}::{file}:{L} - {m}{n}" - rolling: - kind: size + policy: + kind: compound trigger: - max_size: 100mb - policy: - kind: compound - trigger: - kind: fixed_window - pattern: "/tmp/twba/logs/archive/downloader.{}.log" - count: 5 + kind: size + limit: 1 gb + roller: + kind: fixed_window + pattern: "/tmp/twba/logs/archive/downloader.trace.{}.log" + count: 5 + + info_file: + kind: rolling_file + path: "/tmp/twba/logs/downloader.info.log" + filters: + - kind: threshold + level: info + encoder: + pattern: "[{d(%Y-%m-%d %H:%M:%S)}] [{h({l:5})}] {M} - {m}{n}" + policy: + kind: compound + trigger: + kind: size + limit: 100mb + roller: + kind: fixed_window + pattern: "/tmp/twba/logs/archive/downloader.info.{}.log" + count: 5 + + debug_file: + kind: rolling_file + path: "/tmp/twba/logs/downloader.debug.log" + filters: + - kind: threshold + level: debug + encoder: + pattern: "[{d:35}] [{h({l:5})}] {M}::{file}:{L} - {m}{n}" + policy: + kind: compound + trigger: + kind: size + limit: 1gb + roller: + kind: fixed_window + pattern: "/tmp/twba/logs/archive/downloader.debug.{}.log" + count: 5 root: - level: debug + level: trace appenders: - stdout - - file + - debug_file + - trace_file + - info_file \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 4e270f8..e19f9ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -260,9 +260,10 @@ async fn upload_video_to_youtube<'a>( info!("Video has {} parts", part_count); for (i, path) in video_path.iter().enumerate() { info!("Uploading part {} of {}", i + 1, part_count); - let title = get_video_title_from_twitch_video(&video, i, part_count)?; + let title = get_video_title_from_twitch_video(&video, i + 1, part_count)?; info!("youtube part Title: {}", title); - let description = get_video_description_from_twitch_video(&video, i, part_count, &config)?; + let description = + get_video_description_from_twitch_video(&video, i + 1, part_count, &config)?; let privacy = match video.streamer.public_videos_default { Some(true) => PrivacyStatus::Public, @@ -319,13 +320,27 @@ pub async fn split_video_into_parts( let file_playlist = clean(Path::join(&parent_dir, "output.m3u8")); //endregion info!( - "Splitting video: {:?}\n\tinto parts with soft cap duration: {} minutes and hard cap duration: {} minutes", + "Splitting video: {:?} into parts with soft cap duration: {} minutes and hard cap duration: {} minutes", filepath, duration_soft_cap.num_minutes(), duration_hard_cap.num_minutes() ); - let output_path_pattern = format!("{}_%03d.mp4", filepath.to_str().unwrap()); //TODO: maybe make the number of digits dynamic + let output_path_pattern = Path::join( + &parent_dir, + format!( + "{}_%03d.mp4", + filepath + .file_stem() + .expect("could not get file_stem from path") + .to_str() + .expect("could not convert file_stem to str") + ), + ) + .to_str() + .expect("could not convert path to string") + .to_string(); //TODO: maybe make the number of digits dynamic + debug!("output path pattern: {}", output_path_pattern); let duration_str = duration_to_string(&duration_soft_cap); //region run ffmpeg split command @@ -593,7 +608,9 @@ pub fn get_video_title_from_twitch_video( get_video_prefix_from_twitch_video(video, part, total_parts)? ), }; - let title = get_playlist_title_from_twitch_video(video)?; + + let title = video.video.title.as_ref().ok_or("Video has no title")?; + let title = cap_long_title(title)?; let res = format!("{}{}", prefix, title); Ok(res) @@ -605,13 +622,20 @@ const PREFIX_LENGTH: usize = 24; pub fn get_playlist_title_from_twitch_video(video: &data::VideoData) -> Result { trace!("get playlist title from twitch video"); let title = video.video.title.as_ref().ok_or("Video has no title")?; + let date_str = get_date_string_from_video(video)?; + let title = format!("{} {}", date_str, title,); + let title = cap_long_title(title)?; + Ok(title) +} +pub fn cap_long_title>(title: S) -> Result { + let title = title.into(); const SEPARATOR_LEN: usize = 1; if title.len() > MAX_VIDEO_TITLE_LENGTH - PREFIX_LENGTH - SEPARATOR_LEN { - let res = format!( + let shortened = format!( "{}...", &title[0..MAX_VIDEO_TITLE_LENGTH - PREFIX_LENGTH - SEPARATOR_LEN - 3] ); - return Ok(res); + return Ok(shortened); } Ok(title.to_string()) } @@ -625,18 +649,22 @@ pub fn get_video_prefix_from_twitch_video( info!("video: {:?}", video); info!("video.video: {:?}", video.video); info!("video.video.created_at: {:?}", video.video.created_at); + let res = get_date_string_from_video(video)?; + let res = format!("{}[Part {:0>2}/{:0>2}]", res, part, total_parts); + Ok(res) +} + +fn get_date_string_from_video(video: &VideoData) -> Result { let created_at = video .video .created_at .ok_or(format!("Video has no created_at time: {:?}", video.video).as_str())?; // let created_at = created_at.format("%Y-%m-%d"); let res = format!( - "[{:0>4}-{:0>2}-{:0>2}][Part {:0>2}/{:0>2}]", + "[{:0>4}-{:0>2}-{:0>2}]", created_at.year(), created_at.month(), created_at.day(), - part, - total_parts ); Ok(res) } diff --git a/src/main.rs b/src/main.rs index 8d9acd2..40e0601 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,7 +74,16 @@ async fn initialize_logger2() -> Result<(), Box> { // .unwrap(); // // let _handle = log4rs::init_config(config).unwrap(); - log4rs::init_file("logger.yaml", Default::default()).unwrap(); + let logger_config_path = Path::new("logger.yaml"); + let path = &logger_config_path + .canonicalize() + .expect(format!("could not find the file: {:?}", logger_config_path).as_str()); + println!( + "Log config file path: {:?} => {:?}", + logger_config_path, path + ); + log4rs::init_file(path, Default::default()) + .expect("Failed to initialize the logger from the file"); info!("=================================================================================="); info!( "Start of new log on {}", diff --git a/tests/lib_tests.rs b/tests/lib_tests.rs index 5b78a9f..f81a7eb 100644 --- a/tests/lib_tests.rs +++ b/tests/lib_tests.rs @@ -1,12 +1,10 @@ use std::path::{Path, PathBuf}; -use std::sync::Once; use chrono::{DateTime, NaiveDateTime, Utc}; // use bigquery_googleapi::BigqueryClient; use google_bigquery::BigqueryClient; -use log::info; use log::LevelFilter; -use simplelog::{ColorChoice, TermLogger, TerminalMode}; +use log::{debug, info}; use downloader; use downloader::data::{Streamers, VideoData, VideoMetadata, Videos}; @@ -15,17 +13,11 @@ use downloader::{ get_video_title_from_twitch_video, }; -static INIT: Once = Once::new(); fn init_console_logging(log_level: LevelFilter) { - INIT.call_once(|| { - TermLogger::init( - log_level, - simplelog::Config::default(), - TerminalMode::Mixed, - ColorChoice::Auto, - ) - .unwrap(); - }); + let _ = env_logger::builder() + .filter_level(log_level) + .is_test(true) + .try_init(); } async fn get_sample_client() -> BigqueryClient { @@ -94,7 +86,7 @@ async fn get_video_title() { video.video.title = Some(LONG_TITLE.to_string()); let title = get_video_title_from_twitch_video(&video, 5, 20).unwrap(); - info!("part title:\n{}", title); + info!("part title: {}", title); assert_eq!(title, "[2021-01-01][Part 05/20] long title with over a hundred characters that is definitely going to be..."); } @@ -109,7 +101,7 @@ async fn get_video_title_single_part() { video.video.title = Some(LONG_TITLE.to_string()); let title = get_video_title_from_twitch_video(&video, 1, 1).unwrap(); - info!("single part title:\n{}", title); + info!("single part title: {}", title); assert_eq!( title, "long title with over a hundred characters that is definitely going to be..." @@ -123,14 +115,14 @@ async fn get_playlist_title() { let mut video = get_sample_video(&client); let title = get_playlist_title_from_twitch_video(&video).unwrap(); - assert_eq!(title, "Test Video"); + assert_eq!("[2021-01-01] Test Video", title); video.video.title = Some(LONG_TITLE.to_string()); let title = get_playlist_title_from_twitch_video(&video).unwrap(); - info!("playlist title:\n{}", title); + info!("playlist title: {}", title); assert_eq!( - title, - "long title with over a hundred characters that is definitely going to be..." + "[2021-01-01] long title with over a hundred characters that is definitel...", + title ); } @@ -141,7 +133,7 @@ async fn get_video_prefix() { let video = get_sample_video(&client); let prefix = get_video_prefix_from_twitch_video(&video, 5, 20).unwrap(); - info!("prefix:\n{}", prefix); + info!("prefix: {}", prefix); assert_eq!(prefix, "[2021-01-01][Part 05/20]"); } @@ -162,8 +154,14 @@ async fn split_video_into_parts_with_join() { //endregion let parts = parts.expect("failed to split video into parts"); - info!("parts: {:?}", parts); + debug!("parts: {:?}", parts); assert_eq!(5, parts.len(),); + for (i, part) in parts.iter().enumerate() { + assert_eq!( + format!("short_video_00{}.mp4", i), + part.file_name().unwrap().to_str().unwrap() + ); + } } #[tokio::test] @@ -183,8 +181,14 @@ async fn split_video_into_parts_without_join() { //endregion let parts = parts.expect("failed to split video into parts"); - info!("parts: {:?}", parts); + debug!("parts: {:?}", parts); assert_eq!(6, parts.len(),); + for (i, part) in parts.iter().enumerate() { + assert_eq!( + format!("short_video_00{}.mp4", i), + part.file_name().unwrap().to_str().unwrap() + ); + } } fn prepare_existing_video_test_data(temp_subname: i32) -> (PathBuf, PathBuf) {