From 3e33439978d39549bbaee029425bb60f17d47cf4 Mon Sep 17 00:00:00 2001 From: OMGeeky Date: Fri, 9 Jun 2023 00:25:26 +0200 Subject: [PATCH] fix bug where only part of the file is written when combining ts files --- Cargo.toml | 2 +- src/lib.rs | 158 +++++++++++++++++++++++++++++------------------------ 2 files changed, 89 insertions(+), 71 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5bcbcff..1fef104 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "twitch_data" -version = "0.2.2" +version = "0.2.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/lib.rs b/src/lib.rs index aba7df4..77fdbf5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::error::Error; use std::fmt; use std::fmt::{Debug, Display, Formatter}; -use std::io::Write; use std::path::{Path, PathBuf}; use chrono::{DateTime, Utc}; @@ -11,7 +10,7 @@ use exponential_backoff::twitch::{ check_backoff_twitch_get, check_backoff_twitch_get_with_client, check_backoff_twitch_with_client, }; -use futures::future::join_all; + use futures::StreamExt; use reqwest; use serde::{Deserialize, Serialize}; @@ -29,6 +28,7 @@ pub use twitch_types::{UserId, VideoId}; use crate::prelude::*; pub mod prelude; + //region DownloadError #[derive(Debug, Clone)] pub struct DownloadError { @@ -42,6 +42,7 @@ impl Display for DownloadError { } impl Error for DownloadError {} + impl DownloadError { pub fn new>(message: S) -> DownloadError { let message = message.into(); @@ -90,6 +91,7 @@ pub fn convert_twitch_video_to_twitch_data_video(twitch_video: TwitchVideo) -> V thumbnail_url: twitch_video.thumbnail_url, } } + impl<'a> TwitchClient<'a> { pub async fn get_videos_from_login( &self, @@ -106,6 +108,7 @@ impl<'a> TwitchClient<'a> { Ok(v) } } + //endregion Proxies pub enum VideoQuality { Source, @@ -116,6 +119,7 @@ pub enum VideoQuality { AudioOnly, Other(String), } + pub struct TwitchClient<'a> { reqwest_client: reqwest::Client, client: twitch_api::TwitchClient<'a, reqwest::Client>, @@ -461,76 +465,12 @@ impl<'a> TwitchClient<'a> { let mut files: Vec = files.into_iter().map(|f| f.unwrap()).collect(); - files.sort_by_key(|f| { - let number = f - .file_name() - .unwrap() - .to_str() - .unwrap() - .replace("-muted", "") //remove the muted for the sort if its there - .replace(".ts", "") //remove the file ending for the sort - ; + sort_video_part_filenames(&video_id, &mut files); - match number.parse::() { - Ok(n) => n, - Err(e) => { - warn!( - "potentially catchable error while parsing the file number: {}\n{}", - number, e - ); - if !number.starts_with(&format!("{}v", video_id)) || !number.contains("-") { - panic!("Error while parsing the file number: {}", number) - } - let number = number.split("-").collect::>()[1]; - number - .parse() - .expect(format!("Error while parsing the file number: {}", number).as_str()) - } - } - }); - - debug!("combining all parts of video"); let video_ts = output_folder_path.join(&video_id).join("video.ts"); - let mut video_ts_file = tokio::fs::File::create(&video_ts).await?; - for file_path in &files { - // trace!("{:?}", file_path); - let file = tokio::fs::read(&file_path).await?; - // trace!("size of file: {}", file.len()); - video_ts_file.write(&file).await?; - tokio::fs::remove_file(&file_path).await?; - } - - //convert to mp4 - info!("converting to mp4"); let video_mp4 = output_folder_path.join(&video_id).join("video.mp4"); - if video_mp4.exists() { - std::fs::remove_file(&video_mp4)?; - } - debug!( - "running ffmpeg command: ffmpeg -i {} -c copy {}", - video_ts.display(), - video_mp4.display() - ); - let mut cmd = Command::new("ffmpeg"); - let convert_start_time = Instant::now(); - cmd.arg("-i") - .arg(&video_ts) - .arg("-c") - .arg("copy") - .arg(&video_mp4); - let result = cmd.output().await; - //stop the time how long it takes to convert - let duration = Instant::now().duration_since(convert_start_time); - if let Err(e) = result { - error!( - "Error while running ffmpeg command after {:?}: {}", - duration, e - ); - return Err(e.into()); - } - debug!("ffmpeg command finished"); - info!("duration: {:?}", duration); - + combine_parts_into_single_ts(files, &video_ts).await?; + convert_ts_to_mp4(&video_mp4, &video_ts).await?; info!("done converting to mp4"); debug!("removing temporary files"); @@ -708,6 +648,82 @@ pub async fn get_client<'a>() -> Result> { //region static functions +pub fn sort_video_part_filenames(video_id: &str, files: &mut Vec) { + files.sort_by_key(|f| { + let number = f + .file_name() + .unwrap() + .to_str() + .unwrap() + .replace("-muted", "") //remove the muted for the sort if its there + .replace(".ts", "") //remove the file ending for the sort + ; + + match number.parse::() { + Ok(n) => n, + Err(e) => { + warn!( + "potentially catchable error while parsing the file number: {}\n{}", + number, e + ); + if !number.starts_with(&format!("{}v", video_id)) || !number.contains("-") { + panic!("Error while parsing the file number: {}", number) + } + let number = number.split("-").collect::>()[1]; + number + .parse() + .expect(format!("Error while parsing the file number: {}", number).as_str()) + } + } + }); +} +pub async fn convert_ts_to_mp4(video_mp4: &PathBuf, video_ts: &PathBuf) -> Result<()> { + //convert to mp4 + info!("converting to mp4"); + if video_mp4.exists() { + std::fs::remove_file(&video_mp4)?; + } + debug!( + "running ffmpeg command: ffmpeg -i {} -c copy {}", + video_ts.display(), + video_mp4.display() + ); + let mut cmd = Command::new("ffmpeg"); + let convert_start_time = Instant::now(); + cmd.arg("-i") + .arg(&video_ts) + .arg("-c") + .arg("copy") + .arg(&video_mp4); + let result = cmd.output().await; + //stop the time how long it takes to convert + let duration = Instant::now().duration_since(convert_start_time); + if let Err(e) = result { + error!( + "Error while running ffmpeg command after {:?}: {}", + duration, e + ); + return Err(e.into()); + } + debug!("ffmpeg command finished"); + info!("duration: {:?}", duration); + Ok(()) +} + +pub async fn combine_parts_into_single_ts(files: Vec, video_ts: &PathBuf) -> Result<()> { + debug!("combining all parts of video"); + debug!("part amount: {}", files.len()); + let mut video_ts_file = tokio::fs::File::create(&video_ts).await?; + for file_path in &files { + debug!("{:?}", file_path.file_name()); + let file = tokio::fs::read(&file_path).await?; + trace!("size of file: {}", file.len()); + video_ts_file.write_all(&file).await?; + tokio::fs::remove_file(&file_path).await?; + } + Ok(()) +} + fn get_base_url(url: &str) -> String { let mut base_url = url.to_string(); let mut i = base_url.len() - 1; @@ -763,7 +779,7 @@ fn convert_twitch_duration(duration: &str) -> chrono::Duration { }; debug!("hours: {}, mins: {}, secs: {}", hours, mins, secs); - let millis =/* millis +*/ secs*1000 + mins*60*1000 + hours*60*60*1000; + let millis = /* millis +*/ secs * 1000 + mins * 60 * 1000 + hours * 60 * 60 * 1000; let res = chrono::Duration::milliseconds(millis); res @@ -799,11 +815,13 @@ fn convert_twitch_time_info(res: String, fmt: &str) -> DateTime { pub struct TwitchVideoAccessTokenResponse { pub data: VideoAccessTokenResponseData, } + //noinspection ALL #[derive(Debug, Deserialize, Serialize)] pub struct VideoAccessTokenResponseData { pub videoPlaybackAccessToken: VideoAccessTokenResponseDataAccessToken, } + #[derive(Debug, Deserialize, Serialize)] pub struct VideoAccessTokenResponseDataAccessToken { pub value: String,