change logging & some youtube title stuff

2025-12-26 16:07:43 +01:00 · 2023-04-09 18:19:59 +02:00
parent 1c0769b9e4
commit 85e689f413
8 changed files with 199 additions and 48 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,5 @@
 # ingore everything
 *
 # include the build
-!build/downloader
+!build/downloader
+!logger.yaml
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,15 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

+[[package]]
+name = "aho-corasick"
+version = "0.7.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "aliri_braid"
 version = "0.3.1"
@@ -359,6 +368,7 @@ version = "0.1.4"
 dependencies = [
 "chrono",
 "downloader_config",
+ "env_logger",
 "google-bigquery2",
 "google_bigquery",
 "google_youtube",
@@ -401,6 +411,19 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "env_logger"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
+dependencies = [
+ "humantime",
+ "is-terminal",
+ "log 0.4.17",
+ "regex",
+ "termcolor",
+]
+
 [[package]]
 name = "errno"
 version = "0.3.0"
@@ -952,6 +975,18 @@ version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f"

+[[package]]
+name = "is-terminal"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "io-lifetimes",
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1402,6 +1437,23 @@ dependencies = [
 "bitflags",
 ]

+[[package]]
+name = "regex"
+version = "1.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
 [[package]]
 name = "reqwest"
 version = "0.11.16"
@@ -2377,6 +2429,15 @@ dependencies = [
 "windows-targets 0.42.2",
 ]

+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.0",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.42.2"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,3 +20,4 @@ log4rs = { version = "1.2.0" , features = ["compound_policy", "default", "size_t
 path-clean = "1.0.1"

 log-panics = { version = "2", features = ["with-backtrace"]}
+env_logger = "0.10.0"
--- a/2
+++ b/2
@@ -10,6 +10,8 @@ RUN apk add libgcc
 # add ffmpeg to the container (needed for video splitting)
 RUN apk add ffmpeg

+COPY ./logger.yaml ./logger.yaml
+
 # copy the binary from the build folder
 # this binary should be build with the
 # target x86_64-unknown-linux-musl to be able to run
--- a/logger.yaml
+++ b/logger.yaml
@@ -3,25 +3,70 @@ refresh_rate: 30 seconds
 appenders:
  stdout:
    kind: console
+    filters:
+      - kind: threshold
+        level: info
+    encoder:
+      pattern: "[{d(%Y-%m-%d %H:%M:%S)}] [{h({l:5})}] {M} - {m}{n}"

-  requests:
-    kind: file
-    path: "/tmp/twba/logs/downloader.log"
+  trace_file:
+    kind: rolling_file
+    path: "/tmp/twba/logs/downloader.trace.log"
+    filters:
+      - kind: threshold
+        level: trace
    encoder:
      pattern: "[{d:35}] [{h({l:5})}] {M}::{file}:{L} - {m}{n}"
-    rolling:
-      kind: size
+    policy:
+      kind: compound
      trigger:
-        max_size: 100mb
-      policy:
-        kind: compound
-        trigger:
-          kind: fixed_window
-          pattern: "/tmp/twba/logs/archive/downloader.{}.log"
-          count: 5
+        kind: size
+        limit: 1 gb
+      roller:
+        kind: fixed_window
+        pattern: "/tmp/twba/logs/archive/downloader.trace.{}.log"
+        count: 5
+
+  info_file:
+    kind: rolling_file
+    path: "/tmp/twba/logs/downloader.info.log"
+    filters:
+      - kind: threshold
+        level: info
+    encoder:
+      pattern: "[{d(%Y-%m-%d %H:%M:%S)}] [{h({l:5})}] {M} - {m}{n}"
+    policy:
+      kind: compound
+      trigger:
+        kind: size
+        limit: 100mb
+      roller:
+        kind: fixed_window
+        pattern: "/tmp/twba/logs/archive/downloader.info.{}.log"
+        count: 5
+
+  debug_file:
+    kind: rolling_file
+    path: "/tmp/twba/logs/downloader.debug.log"
+    filters:
+      - kind: threshold
+        level: debug
+    encoder:
+      pattern: "[{d:35}] [{h({l:5})}] {M}::{file}:{L} - {m}{n}"
+    policy:
+      kind: compound
+      trigger:
+        kind: size
+        limit: 1gb
+      roller:
+        kind: fixed_window
+        pattern: "/tmp/twba/logs/archive/downloader.debug.{}.log"
+        count: 5

 root:
-  level: debug
+  level: trace
  appenders:
    - stdout
-    - file
+    - debug_file
+    - trace_file
+    - info_file
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -260,9 +260,10 @@ async fn upload_video_to_youtube<'a>(
    info!("Video has {} parts", part_count);
    for (i, path) in video_path.iter().enumerate() {
        info!("Uploading part {} of {}", i + 1, part_count);
-        let title = get_video_title_from_twitch_video(&video, i, part_count)?;
+        let title = get_video_title_from_twitch_video(&video, i + 1, part_count)?;
        info!("youtube part Title: {}", title);
-        let description = get_video_description_from_twitch_video(&video, i, part_count, &config)?;
+        let description =
+            get_video_description_from_twitch_video(&video, i + 1, part_count, &config)?;

        let privacy = match video.streamer.public_videos_default {
            Some(true) => PrivacyStatus::Public,
@@ -319,13 +320,27 @@ pub async fn split_video_into_parts(
    let file_playlist = clean(Path::join(&parent_dir, "output.m3u8"));
    //endregion
    info!(
-        "Splitting video: {:?}\n\tinto parts with soft cap duration: {} minutes and hard cap duration: {} minutes",
+        "Splitting video: {:?} into parts with soft cap duration: {} minutes and hard cap duration: {} minutes",
        filepath,
        duration_soft_cap.num_minutes(),
        duration_hard_cap.num_minutes()
    );

-    let output_path_pattern = format!("{}_%03d.mp4", filepath.to_str().unwrap()); //TODO: maybe make the number of digits dynamic
+    let output_path_pattern = Path::join(
+        &parent_dir,
+        format!(
+            "{}_%03d.mp4",
+            filepath
+                .file_stem()
+                .expect("could not get file_stem from path")
+                .to_str()
+                .expect("could not convert file_stem to str")
+        ),
+    )
+    .to_str()
+    .expect("could not convert path to string")
+    .to_string(); //TODO: maybe make the number of digits dynamic
+    debug!("output path pattern: {}", output_path_pattern);
    let duration_str = duration_to_string(&duration_soft_cap);

    //region run ffmpeg split command
@@ -593,7 +608,9 @@ pub fn get_video_title_from_twitch_video(
            get_video_prefix_from_twitch_video(video, part, total_parts)?
        ),
    };
-    let title = get_playlist_title_from_twitch_video(video)?;
+
+    let title = video.video.title.as_ref().ok_or("Video has no title")?;
+    let title = cap_long_title(title)?;

    let res = format!("{}{}", prefix, title);
    Ok(res)
@@ -605,13 +622,20 @@ const PREFIX_LENGTH: usize = 24;
 pub fn get_playlist_title_from_twitch_video(video: &data::VideoData) -> Result<String> {
    trace!("get playlist title from twitch video");
    let title = video.video.title.as_ref().ok_or("Video has no title")?;
+    let date_str = get_date_string_from_video(video)?;
+    let title = format!("{} {}", date_str, title,);
+    let title = cap_long_title(title)?;
+    Ok(title)
+}
+pub fn cap_long_title<S: Into<String>>(title: S) -> Result<String> {
+    let title = title.into();
    const SEPARATOR_LEN: usize = 1;
    if title.len() > MAX_VIDEO_TITLE_LENGTH - PREFIX_LENGTH - SEPARATOR_LEN {
-        let res = format!(
+        let shortened = format!(
            "{}...",
            &title[0..MAX_VIDEO_TITLE_LENGTH - PREFIX_LENGTH - SEPARATOR_LEN - 3]
        );
-        return Ok(res);
+        return Ok(shortened);
    }
    Ok(title.to_string())
 }
@@ -625,18 +649,22 @@ pub fn get_video_prefix_from_twitch_video(
    info!("video: {:?}", video);
    info!("video.video: {:?}", video.video);
    info!("video.video.created_at: {:?}", video.video.created_at);
+    let res = get_date_string_from_video(video)?;
+    let res = format!("{}[Part {:0>2}/{:0>2}]", res, part, total_parts);
+    Ok(res)
+}
+
+fn get_date_string_from_video(video: &VideoData) -> Result<String> {
    let created_at = video
        .video
        .created_at
        .ok_or(format!("Video has no created_at time: {:?}", video.video).as_str())?;
    // let created_at = created_at.format("%Y-%m-%d");
    let res = format!(
-        "[{:0>4}-{:0>2}-{:0>2}][Part {:0>2}/{:0>2}]",
+        "[{:0>4}-{:0>2}-{:0>2}]",
        created_at.year(),
        created_at.month(),
        created_at.day(),
-        part,
-        total_parts
    );
    Ok(res)
 }
--- a/src/main.rs
+++ b/src/main.rs
@@ -74,7 +74,16 @@ async fn initialize_logger2() -> Result<(), Box<dyn Error>> {
    //     .unwrap();
    //
    // let _handle = log4rs::init_config(config).unwrap();
-    log4rs::init_file("logger.yaml", Default::default()).unwrap();
+    let logger_config_path = Path::new("logger.yaml");
+    let path = &logger_config_path
+        .canonicalize()
+        .expect(format!("could not find the file: {:?}", logger_config_path).as_str());
+    println!(
+        "Log config file path: {:?} => {:?}",
+        logger_config_path, path
+    );
+    log4rs::init_file(path, Default::default())
+        .expect("Failed to initialize the logger from the file");
    info!("==================================================================================");
    info!(
        "Start of new log on {}",
--- a/tests/lib_tests.rs
+++ b/tests/lib_tests.rs
@@ -1,12 +1,10 @@
 use std::path::{Path, PathBuf};
-use std::sync::Once;

 use chrono::{DateTime, NaiveDateTime, Utc};
 // use bigquery_googleapi::BigqueryClient;
 use google_bigquery::BigqueryClient;
-use log::info;
 use log::LevelFilter;
-use simplelog::{ColorChoice, TermLogger, TerminalMode};
+use log::{debug, info};

 use downloader;
 use downloader::data::{Streamers, VideoData, VideoMetadata, Videos};
@@ -15,17 +13,11 @@ use downloader::{
    get_video_title_from_twitch_video,
 };

-static INIT: Once = Once::new();
 fn init_console_logging(log_level: LevelFilter) {
-    INIT.call_once(|| {
-        TermLogger::init(
-            log_level,
-            simplelog::Config::default(),
-            TerminalMode::Mixed,
-            ColorChoice::Auto,
-        )
-        .unwrap();
-    });
+    let _ = env_logger::builder()
+        .filter_level(log_level)
+        .is_test(true)
+        .try_init();
 }

 async fn get_sample_client() -> BigqueryClient {
@@ -94,7 +86,7 @@ async fn get_video_title() {

    video.video.title = Some(LONG_TITLE.to_string());
    let title = get_video_title_from_twitch_video(&video, 5, 20).unwrap();
-    info!("part title:\n{}", title);
+    info!("part title: {}", title);
    assert_eq!(title, "[2021-01-01][Part 05/20] long title with over a hundred characters that is definitely going to be...");
 }

@@ -109,7 +101,7 @@ async fn get_video_title_single_part() {

    video.video.title = Some(LONG_TITLE.to_string());
    let title = get_video_title_from_twitch_video(&video, 1, 1).unwrap();
-    info!("single part title:\n{}", title);
+    info!("single part title: {}", title);
    assert_eq!(
        title,
        "long title with over a hundred characters that is definitely going to be..."
@@ -123,14 +115,14 @@ async fn get_playlist_title() {
    let mut video = get_sample_video(&client);

    let title = get_playlist_title_from_twitch_video(&video).unwrap();
-    assert_eq!(title, "Test Video");
+    assert_eq!("[2021-01-01] Test Video", title);

    video.video.title = Some(LONG_TITLE.to_string());
    let title = get_playlist_title_from_twitch_video(&video).unwrap();
-    info!("playlist title:\n{}", title);
+    info!("playlist title: {}", title);
    assert_eq!(
-        title,
-        "long title with over a hundred characters that is definitely going to be..."
+        "[2021-01-01] long title with over a hundred characters that is definitel...",
+        title
    );
 }

@@ -141,7 +133,7 @@ async fn get_video_prefix() {
    let video = get_sample_video(&client);

    let prefix = get_video_prefix_from_twitch_video(&video, 5, 20).unwrap();
-    info!("prefix:\n{}", prefix);
+    info!("prefix: {}", prefix);
    assert_eq!(prefix, "[2021-01-01][Part 05/20]");
 }

@@ -162,8 +154,14 @@ async fn split_video_into_parts_with_join() {
    //endregion

    let parts = parts.expect("failed to split video into parts");
-    info!("parts: {:?}", parts);
+    debug!("parts: {:?}", parts);
    assert_eq!(5, parts.len(),);
+    for (i, part) in parts.iter().enumerate() {
+        assert_eq!(
+            format!("short_video_00{}.mp4", i),
+            part.file_name().unwrap().to_str().unwrap()
+        );
+    }
 }

 #[tokio::test]
@@ -183,8 +181,14 @@ async fn split_video_into_parts_without_join() {
    //endregion

    let parts = parts.expect("failed to split video into parts");
-    info!("parts: {:?}", parts);
+    debug!("parts: {:?}", parts);
    assert_eq!(6, parts.len(),);
+    for (i, part) in parts.iter().enumerate() {
+        assert_eq!(
+            format!("short_video_00{}.mp4", i),
+            part.file_name().unwrap().to_str().unwrap()
+        );
+    }
 }

 fn prepare_existing_video_test_data(temp_subname: i32) -> (PathBuf, PathBuf) {