extract stuff to common

Cargo.lock

@@ -529,6 +529,15 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.11"
@@ -2048,7 +2057,6 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls 0.24.2",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -2058,7 +2066,6 @@ dependencies = [
  "once_cell",
  "percent-encoding 2.3.1",
  "pin-project-lite",
- "rustls 0.21.11",
  "rustls-pemfile 1.0.4",
  "serde",
  "serde_json",
@@ -2067,7 +2074,6 @@ dependencies = [
  "system-configuration",
  "tokio",
  "tokio-native-tls",
- "tokio-rustls 0.24.1",
  "tokio-util",
  "tower-service",
  "url 2.5.0",
@@ -2075,7 +2081,6 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots",
  "winreg",
 ]
 
@@ -2167,9 +2172,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
 [[package]]
 name = "rustix"
-version = "0.38.32"
+version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
  "bitflags 2.5.0",
  "errno",
@@ -2199,7 +2204,7 @@ dependencies = [
  "log",
  "ring",
  "rustls-pki-types",
- "rustls-webpki 0.102.2",
+ "rustls-webpki 0.102.3",
  "subtle",
  "zeroize",
 ]
@@ -2250,9 +2255,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.4.1"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247"
+checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54"
 
 [[package]]
 name = "rustls-webpki"
@@ -2266,9 +2271,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.2"
+version = "0.102.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610"
+checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf"
 dependencies = [
  "ring",
  "rustls-pki-types",
@@ -3295,6 +3300,18 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-appender"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
+dependencies = [
+ "crossbeam-channel",
+ "thiserror",
+ "time",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-attributes"
 version = "0.1.27"
@@ -3353,11 +3370,25 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "twba-backup-config"
-version = "0.1.3"
+version = "0.1.6"
-source = "git+https://github.com/OMGeeky/backup_config.git#88645945830d3076478827503fef43541a2f047e"
+source = "git+https://github.com/OMGeeky/backup_config.git#60092672b4b45667ef21073b3c43c062715ce486"
 dependencies = [
  "confique",
  "serde",
+ "shellexpand",
+ "tracing",
+]
+
+[[package]]
+name = "twba-common"
+version = "0.1.0"
+source = "git+https://github.com/OMGeeky/twba.common.git#a3499ea8bf573db344ec8269782ecd576f3291b5"
+dependencies = [
+ "tracing",
+ "tracing-appender",
+ "tracing-subscriber",
+ "twba-backup-config",
+ "twba-local-db",
 ]
 
 [[package]]
@@ -3376,7 +3407,7 @@ dependencies = [
 [[package]]
 name = "twba-twitch-data"
 version = "0.3.2"
-source = "git+https://github.com/OMGeeky/twitch_data.git#64635bef670e4feb6ca423d0da286db52e687965"
+source = "git+https://github.com/OMGeeky/twitch_data.git#ad4fc9a2096bef2637ec400bd1bc79b131274ec7"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -3411,6 +3442,7 @@ dependencies = [
  "tracing",
  "tracing-subscriber",
  "twba-backup-config",
+ "twba-common",
  "twba-local-db",
  "twba-twitch-data",
 ]
@@ -3744,11 +3776,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
+checksum = "134306a13c5647ad6453e8deaec55d3a44d6021970129e6188735e74bf546697"
 dependencies = [
- "winapi",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -3935,9 +3967,9 @@ dependencies = [
 
 [[package]]
 name = "yup-oauth2"
-version = "8.3.3"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45b7ff561fdc7809a2adad8bce73e157d01129074098e6405d0d7dfa2d087782"
+checksum = "17215ed977ad07d9c95df12503b46dc1c7ebe431c4ab78a5172059127e9bc872"
 dependencies = [
  "anyhow",
  "async-trait",

Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 twba-backup-config = { version = "0.1.1", git = "https://github.com/OMGeeky/backup_config.git" }
 twba-local-db = { version = "0.2", git = "https://github.com/OMGeeky/twitch_backup.local_db.git" }
 twba-twitch-data = { version = "0.3", git = "https://github.com/OMGeeky/twitch_data.git" }
+twba-common = { version = "0.1", git = "https://github.com/OMGeeky/twba.common.git" }
 
 tracing-subscriber = "0.3"
 tracing = "0.1"

src/client.rs

@@ -5,6 +5,7 @@ use twba_local_db::entities::videos::ActiveModel;
 use twba_local_db::prelude::{Status, Users, UsersColumn, Videos, VideosColumn};
 use twba_local_db::re_exports::sea_orm::{
     ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter,
+    QuerySelect,
 };
 use twba_twitch_data::TwitchClient;
 
@@ -28,6 +29,7 @@ impl<'a> FetcherClient<'a> {
     pub(crate) async fn fetch_new_videos(&self) -> Result<()> {
         let users = Users::find()
             .filter(UsersColumn::Active.eq(true))
+            .limit(self.conf.max_items_to_process)
             .all(&self.db)
             .await?;
         info!("Fetching videos for {} users", users.len());

src/main.rs

@@ -4,15 +4,11 @@ pub mod prelude;
 use crate::prelude::*;
 use twba_backup_config::prelude::Config;
 use twba_backup_config::Conf;
+use twba_common::{get_config, init_tracing};
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    tracing_subscriber::fmt()
+    let _guard = init_tracing("twba_twitch_fetcher");
-        .with_max_level(tracing::Level::INFO)
-        .with_env_filter(
-            "sea_orm=warn,sea_orm_migration=warn,sqlx=warn,twba_twitch_fetcher=trace,twba_local_db=warn,twba_twitch_data=info,twba_downloader_config=info,twba_backup_config=info,other=warn",
-        )
-        .init();
     info!("Hello, world!");
 
     run().await?;
@@ -22,13 +18,7 @@ async fn main() -> Result<()> {
 }
 
 async fn run() -> Result<()> {
-    let conf = Conf::builder()
+    let conf = get_config();
-        .env()
-        .file("./settings.toml")
-        .file(shellexpand::tilde("~/twba/config.toml").into_owned())
-        .file(std::env::var("TWBA_CONFIG").unwrap_or_else(|_| "~/twba/config.toml".to_string()))
-        .load()
-        .map_err(|e| FetcherError::LoadConfig(e.into()))?;
 
     trace!("Opening database");
     let db = twba_local_db::open_database(Some(&conf.db_url)).await?;