init

.gitignore

@@ -0,0 +1,2 @@
+/target
+/.idea

Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "twitch-fetcher"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+twba-backup-config = { version = "0.1.1", git = "https://github.com/OMGeeky/backup_config.git" }
+twba-local-db = { version = "0.2", git = "https://github.com/OMGeeky/twitch_backup.local_db.git" }
+twba-twitch-data = { version = "0.3", git = "https://github.com/OMGeeky/twitch_data.git" }
+
+tracing-subscriber = "0.3"
+tracing = "0.1"
+tokio = { version = "1.33", features = ["rt", "rt-multi-thread", "macros"] }
+
+shellexpand = "3.1"
+thiserror = "1.0"
+anyhow = "1.0"
+
+chrono = "0.4"

src/client.rs

@@ -0,0 +1,94 @@
+use crate::prelude::*;
+use twba_backup_config::Conf;
+use twba_local_db::entities::users::Model;
+use twba_local_db::entities::videos::ActiveModel;
+use twba_local_db::prelude::{Status, Users, UsersColumn, Videos, VideosColumn, VideosModel};
+use twba_local_db::re_exports::sea_orm::{
+    ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter,
+};
+use twba_twitch_data::TwitchClient;
+
+pub struct FetcherClient<'a> {
+    pub conf: Conf,
+    pub db: DatabaseConnection,
+    pub twitch_client: TwitchClient<'a>,
+}
+
+impl<'a> FetcherClient<'a> {
+    pub(crate) fn new(conf: Conf, db: DatabaseConnection, twitch_client: TwitchClient<'a>) -> Self {
+        Self {
+            conf,
+            db,
+            twitch_client,
+        }
+    }
+}
+
+impl<'a> FetcherClient<'a> {
+    pub(crate) async fn fetch_new_videos(&self) -> Result<()> {
+        let users = Users::find()
+            .filter(UsersColumn::Active.eq(true))
+            .all(&self.db)
+            .await?;
+        info!("Fetching videos for {} users", users.len());
+        for user in users {
+            match self.fetch_videos_for_user(&user).await {
+                Ok(_) => {
+                    info!("Fetched videos for user: {}", user.twitch_name);
+                }
+                Err(e) => {
+                    error!(
+                        "Could not fetch videos for user: {} because of error: {:?}",
+                        user.twitch_name, e
+                    );
+                }
+            }
+        }
+        info!("Done fetching videos for all users");
+        Ok(())
+    }
+
+    async fn fetch_videos_for_user(&self, user: &Model) -> Result<()> {
+        info!("Fetching videos for user: '{}'", user.twitch_name);
+        let videos = self
+            .twitch_client
+            .get_videos_from_login(&user.twitch_id, None)
+            .await
+            .map_err(FetcherError::GetVideosError)?;
+        for video in videos {
+            info!("Adding video: {} to the database", video.title);
+            let existing_video_found = Videos::find()
+                .filter(VideosColumn::TwitchId.eq(video.id.to_string()))
+                .one(&self.db)
+                .await
+                .is_ok();
+            if existing_video_found {
+                info!("Video with id: {} already exists in the database", video.id);
+                continue;
+            }
+
+            ActiveModel {
+                id: ActiveValue::NotSet,
+                twitch_id: ActiveValue::Set(video.id.to_string()),
+                name: ActiveValue::Set(video.title),
+                user_id: ActiveValue::Set(user.id),
+                created_at: ActiveValue::Set(video.created_at.to_rfc3339()),
+                youtube_id: ActiveValue::NotSet,
+                youtube_playlist_name: ActiveValue::NotSet,
+                youtube_preview_image_url: ActiveValue::NotSet,
+                youtube_playlist_id: ActiveValue::NotSet,
+                duration: ActiveValue::Set(video.duration as i32),
+                twitch_download_url: ActiveValue::Set(Some(video.url)),
+                status: ActiveValue::Set(Status::NotStarted),
+                fail_count: ActiveValue::NotSet,
+                part_count: ActiveValue::Set(0),
+                twitch_preview_image_url: ActiveValue::NotSet,
+                youtube_playlist_created_at: ActiveValue::NotSet,
+                fail_reason: ActiveValue::NotSet,
+            }
+            .insert(&self.db)
+            .await?;
+        }
+        Ok(())
+    }
+}

src/errors.rs

@@ -0,0 +1,16 @@
+use std::error::Error as StdError;
+use std::path::PathBuf;
+
+#[derive(Debug, thiserror::Error)]
+pub enum FetcherError {
+    #[error("Could not load config")]
+    LoadConfig(#[source] anyhow::Error),
+    #[error("Some error with the database: {0:?}")]
+    OpenDatabase(#[from] twba_local_db::re_exports::sea_orm::DbErr),
+    #[error("File or Folder not found or invalid: {0:?}")]
+    NotFound(PathBuf),
+    #[error("Error creating client: {0:?}")]
+    CreateClientError(#[source] Box<dyn StdError>),
+    #[error("Could not get videos: {0:?}")]
+    GetVideosError(#[source] Box<dyn StdError>),
+}

src/main.rs

@@ -0,0 +1,41 @@
+mod client;
+pub mod errors;
+pub mod prelude;
+use crate::prelude::*;
+use twba_backup_config::prelude::Config;
+use twba_backup_config::Conf;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::INFO)
+        .with_env_filter(
+            "sea_orm=warn,sea_orm_migration=warn,sqlx=warn,twba_fetcher=trace,twba_local_db=warn,twba_twitch_data=info,twba_downloader_config=info,twba_backup_config=info,other=warn",
+        )
+        .init();
+    info!("Hello, world!");
+
+    run().await?;
+
+    info!("Bye");
+    Ok(())
+}
+
+async fn run() -> Result<()> {
+    let conf = Conf::builder()
+        .env()
+        .file("./settings.toml")
+        .file(shellexpand::tilde("~/twba/config.toml").into_owned())
+        .load()
+        .map_err(|e| FetcherError::LoadConfig(e.into()))?;
+
+    let db = twba_local_db::open_database(Some(&conf.db_url)).await?;
+    twba_local_db::migrate_db(&db).await?;
+    let twitch_client = twba_twitch_data::get_client()
+        .await
+        .map_err(FetcherError::CreateClientError)?;
+
+    let client = client::FetcherClient::new(conf, db, twitch_client);
+    client.fetch_new_videos().await?;
+    Ok(())
+}

src/prelude.rs

@@ -0,0 +1,7 @@
+use twba_backup_config::Conf;
+
+pub use crate::errors::FetcherError;
+pub(crate) use std::result::Result as StdResult;
+pub(crate) use tracing::{debug, error, info, trace, warn};
+
+pub type Result<T> = StdResult<T, FetcherError>;