diff --git a/Cargo.lock b/Cargo.lock index a615bc5..c08d9ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -529,6 +529,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-channel" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -2048,7 +2057,6 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -2058,7 +2066,6 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite", - "rustls 0.21.11", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -2067,7 +2074,6 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", "tokio-util", "tower-service", "url 2.5.0", @@ -2075,7 +2081,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", "winreg", ] @@ -2167,9 +2172,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -2199,7 +2204,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.2", + "rustls-webpki 0.102.3", "subtle", "zeroize", ] @@ -2250,9 +2255,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" [[package]] name = "rustls-webpki" @@ -2266,9 +2271,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ "ring", "rustls-pki-types", @@ -3295,6 +3300,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.27" @@ -3353,11 +3370,25 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twba-backup-config" -version = "0.1.3" -source = "git+https://github.com/OMGeeky/backup_config.git#88645945830d3076478827503fef43541a2f047e" +version = "0.1.6" +source = "git+https://github.com/OMGeeky/backup_config.git#60092672b4b45667ef21073b3c43c062715ce486" dependencies = [ "confique", "serde", + "shellexpand", + "tracing", +] + +[[package]] +name = "twba-common" +version = "0.1.0" +source = "git+https://github.com/OMGeeky/twba.common.git#a3499ea8bf573db344ec8269782ecd576f3291b5" +dependencies = [ + "tracing", + "tracing-appender", + "tracing-subscriber", + "twba-backup-config", + "twba-local-db", ] [[package]] @@ -3376,7 +3407,7 @@ dependencies = [ [[package]] name = "twba-twitch-data" version = "0.3.2" -source = "git+https://github.com/OMGeeky/twitch_data.git#64635bef670e4feb6ca423d0da286db52e687965" +source = "git+https://github.com/OMGeeky/twitch_data.git#ad4fc9a2096bef2637ec400bd1bc79b131274ec7" dependencies = [ "anyhow", "async-recursion", @@ -3411,6 +3442,7 @@ dependencies = [ "tracing", "tracing-subscriber", "twba-backup-config", + "twba-common", "twba-local-db", "twba-twitch-data", ] @@ -3744,11 +3776,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "134306a13c5647ad6453e8deaec55d3a44d6021970129e6188735e74bf546697" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -3935,9 +3967,9 @@ dependencies = [ [[package]] name = "yup-oauth2" -version = "8.3.3" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b7ff561fdc7809a2adad8bce73e157d01129074098e6405d0d7dfa2d087782" +checksum = "17215ed977ad07d9c95df12503b46dc1c7ebe431c4ab78a5172059127e9bc872" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index e7e2510..53a1f7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" twba-backup-config = { version = "0.1.1", git = "https://github.com/OMGeeky/backup_config.git" } twba-local-db = { version = "0.2", git = "https://github.com/OMGeeky/twitch_backup.local_db.git" } twba-twitch-data = { version = "0.3", git = "https://github.com/OMGeeky/twitch_data.git" } +twba-common = { version = "0.1", git = "https://github.com/OMGeeky/twba.common.git" } tracing-subscriber = "0.3" tracing = "0.1" diff --git a/src/client.rs b/src/client.rs index 83a33d1..5e1d1e9 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,6 +5,7 @@ use twba_local_db::entities::videos::ActiveModel; use twba_local_db::prelude::{Status, Users, UsersColumn, Videos, VideosColumn}; use twba_local_db::re_exports::sea_orm::{ ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, + QuerySelect, }; use twba_twitch_data::TwitchClient; @@ -28,6 +29,7 @@ impl<'a> FetcherClient<'a> { pub(crate) async fn fetch_new_videos(&self) -> Result<()> { let users = Users::find() .filter(UsersColumn::Active.eq(true)) + .limit(self.conf.max_items_to_process) .all(&self.db) .await?; info!("Fetching videos for {} users", users.len()); diff --git a/src/main.rs b/src/main.rs index f5f7f07..66f7815 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,15 +4,11 @@ pub mod prelude; use crate::prelude::*; use twba_backup_config::prelude::Config; use twba_backup_config::Conf; +use twba_common::{get_config, init_tracing}; #[tokio::main] async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .with_env_filter( - "sea_orm=warn,sea_orm_migration=warn,sqlx=warn,twba_twitch_fetcher=trace,twba_local_db=warn,twba_twitch_data=info,twba_downloader_config=info,twba_backup_config=info,other=warn", - ) - .init(); + let _guard = init_tracing("twba_twitch_fetcher"); info!("Hello, world!"); run().await?; @@ -22,13 +18,7 @@ async fn main() -> Result<()> { } async fn run() -> Result<()> { - let conf = Conf::builder() - .env() - .file("./settings.toml") - .file(shellexpand::tilde("~/twba/config.toml").into_owned()) - .file(std::env::var("TWBA_CONFIG").unwrap_or_else(|_| "~/twba/config.toml".to_string())) - .load() - .map_err(|e| FetcherError::LoadConfig(e.into()))?; + let conf = get_config(); trace!("Opening database"); let db = twba_local_db::open_database(Some(&conf.db_url)).await?;