From 035d4e20dd65b0bd5e7cf58cae41cc7eb316895e Mon Sep 17 00:00:00 2001 From: lonelyhentxi Date: Fri, 29 Mar 2024 00:30:00 +0800 Subject: [PATCH] feat: add dal context and fill pull items --- .gitignore | 2 +- Cargo.lock | 837 ++++++++++-------- Cargo.toml | 5 +- config/recorder.development.yaml | 23 - config/recorder.test.yaml | 28 +- crates/recorder/Cargo.toml | 34 +- crates/recorder/src/app.rs | 8 +- crates/recorder/src/config/mod.rs | 2 +- crates/recorder/src/downloaders/api_client.rs | 76 ++ crates/recorder/src/downloaders/defs.rs | 316 +------ crates/recorder/src/downloaders/mod.rs | 5 +- crates/recorder/src/downloaders/qbitorrent.rs | 13 +- crates/recorder/src/downloaders/torrent.rs | 341 +++++++ .../src/downloaders/torrent_downloader.rs | 96 -- crates/recorder/src/lib.rs | 2 +- crates/recorder/src/migrations/defs.rs | 34 +- .../src/migrations/m20220101_000001_init.rs | 70 +- .../m20240224_082543_add_downloads.rs | 27 +- ...240225_060853_subscriber_add_downloader.rs | 2 +- crates/recorder/src/models/bangumi.rs | 31 +- crates/recorder/src/models/db_utils.rs | 2 +- crates/recorder/src/models/downloads.rs | 70 +- .../recorder/src/models/entities/bangumi.rs | 17 +- .../src/models/entities/downloaders.rs | 2 - .../recorder/src/models/entities/downloads.rs | 4 +- .../recorder/src/models/entities/episodes.rs | 9 +- .../src/models/entities/subscribers.rs | 6 +- crates/recorder/src/models/episodes.rs | 58 +- crates/recorder/src/models/subscriptions.rs | 209 ++++- crates/recorder/src/parsers/errors.rs | 2 + .../src/parsers/mikan/mikan_client.rs | 2 +- .../src/parsers/mikan/mikan_ep_parser.rs | 61 +- .../src/parsers/mikan/mikan_rss_parser.rs | 7 + crates/recorder/src/parsers/mikan/mod.rs | 1 + .../recorder/src/parsers/raw/raw_ep_parser.rs | 69 +- .../recorder/src/parsers/tmdb/tmdb_client.rs | 2 +- crates/recorder/src/path/mod.rs | 3 + crates/recorder/src/path/torrent_path.rs | 4 +- crates/recorder/src/path/url_utils.rs | 19 + .../src/{dal/mod.rs => storage/dal.rs} | 37 +- crates/recorder/src/storage/dal_ext.rs | 40 + .../recorder/src/storage/dal_initializer.rs | 26 + crates/recorder/src/storage/mod.rs | 7 + crates/recorder/src/workers/collect.rs | 7 +- crates/recorder/src/workers/subscription.rs | 6 +- 45 files changed, 1561 insertions(+), 1061 deletions(-) create mode 100644 crates/recorder/src/downloaders/api_client.rs create mode 100644 crates/recorder/src/downloaders/torrent.rs delete mode 100644 crates/recorder/src/downloaders/torrent_downloader.rs create mode 100644 crates/recorder/src/path/url_utils.rs rename crates/recorder/src/{dal/mod.rs => storage/dal.rs} (67%) create mode 100644 crates/recorder/src/storage/dal_ext.rs create mode 100644 crates/recorder/src/storage/dal_initializer.rs create mode 100644 crates/recorder/src/storage/mod.rs diff --git a/.gitignore b/.gitignore index 6ff0429..9fdca61 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,7 @@ coverage # nyc tests coverage .nyc_output -# Grunt intermediate dal (https://gruntjs.com/creating-plugins#storing-task-files) +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) diff --git a/Cargo.lock b/Cargo.lock index 3c71b3d..20af2c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "getrandom", @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -105,9 +105,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.12" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -153,9 +153,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" [[package]] name = "argon2" @@ -219,7 +219,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -235,13 +235,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -266,16 +266,6 @@ dependencies = [ "quick-xml 0.30.0", ] -[[package]] -name = "atomic-write-file" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" -dependencies = [ - "nix", - "rand", -] - [[package]] name = "auto-future" version = "1.0.0" @@ -284,22 +274,22 @@ checksum = "3c1e7e457ea78e524f48639f551fd79703ac3f2237f5ecccdf4708f8a75ad373" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "axum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core", "axum-macros", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "hyper 1.2.0", @@ -315,7 +305,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.0", "tokio", "tower", "tower-layer", @@ -332,13 +322,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", "tracing", @@ -346,16 +336,16 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895ff42f72016617773af68fb90da2a9677d89c62338ec09162d4909d86fdd8f" +checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ "axum", "axum-core", "bytes", "cookie", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "mime", @@ -364,6 +354,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -372,17 +363,17 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00c055ee2d014ae5981ce1016374e8213682aa14d9bf40e48ab48b5f3ef20eaa" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] name = "axum-test" -version = "14.3.0" +version = "14.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc431b62ab307c833af24700936485eb5f9a8ac18a19347fe37dd4f7ae3dffe9" +checksum = "feeac32f78598625ed66da8eb352354f1a959311a029fa418d6eed17551eb786" dependencies = [ "anyhow", "async-trait", @@ -390,7 +381,7 @@ dependencies = [ "axum", "bytes", "cookie", - "http 1.0.0", + "http 1.1.0", "http-body-util", "hyper 1.2.0", "hyper-util", @@ -423,9 +414,9 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79c8ef183b8b663e8cb19cf92fb7d98c56739977bd47eae2de2717bd5de2c2c" +checksum = "c491fa80d69c03084223a4e73c378dd9f9a1e612eb54051213f88b2d5249b458" dependencies = [ "fastrand", "futures-core", @@ -435,9 +426,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -472,6 +463,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64-simd" version = "0.7.0" @@ -534,9 +531,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" dependencies = [ "serde", ] @@ -592,9 +589,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58b559fd6448c6e2fd0adb5720cd98a2506594cafa4737ff98c396f3e82f667" +checksum = "0901fc8eb0aca4c83be0106d6f2db17d86a08dfc2c25f0e84464bf381158add6" dependencies = [ "borsh-derive", "cfg_aliases", @@ -602,23 +599,23 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aadb5b6ccbd078890f6d7003694e33816e6b784358f18e15e7e6d9f065a57cd" +checksum = "51670c3aa053938b0ee3bd67c3817e471e626151131b934038e83c5bf8de48f5" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", "syn_derive", ] [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -637,9 +634,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "serde", @@ -653,9 +650,9 @@ checksum = "0d75b8252ed252f881d1dc4482ae3c3854df6ee8183c1906bac50ff358f4f89f" [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "byte-unit" @@ -697,9 +694,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "camino" @@ -712,9 +709,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" dependencies = [ "serde", ] @@ -744,10 +741,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.86" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9fa1897e4325be0d68d48df6aa1a71ac2ed4d27723887e7754192705350730" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ + "jobserver", "libc", ] @@ -765,9 +763,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -775,7 +773,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -822,9 +820,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.1" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -832,9 +830,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.1" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstream", "anstyle", @@ -844,14 +842,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.0" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -928,9 +926,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -968,9 +966,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd91cf61412820176e137621345ee43b3f4423e589e7ae4e50d601d93e35ef8" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" dependencies = [ "percent-encoding", "time", @@ -1163,7 +1161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -1235,7 +1233,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -1268,7 +1266,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core 0.20.8", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -1514,11 +1512,11 @@ dependencies = [ [[package]] name = "email-encoding" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75" +checksum = "a87260449b06739ee78d6281c68d2a0ff3e3af64a78df63d3a1aeb3c06997c8a" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "memchr", ] @@ -1593,15 +1591,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "finl_unicode" @@ -1611,9 +1609,9 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "flagset" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" +checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" [[package]] name = "flate2" @@ -1748,7 +1746,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -1850,8 +1848,8 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -1867,17 +1865,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http 0.2.11", - "indexmap 2.2.3", + "http 0.2.12", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1886,17 +1884,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http 1.0.0", - "indexmap 2.2.3", + "http 1.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1918,7 +1916,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.9", + "ahash 0.8.11", "allocator-api2", ] @@ -1941,10 +1939,16 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.3.6" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2007,9 +2011,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -2018,9 +2022,9 @@ dependencies = [ [[package]] name = "http" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -2034,7 +2038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] @@ -2045,18 +2049,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.0.0", + "http 1.1.0", ] [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", - "futures-util", - "http 1.0.0", + "futures-core", + "http 1.1.0", "http-body 1.0.0", "pin-project-lite", ] @@ -2104,8 +2108,8 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.24", - "http 0.2.11", + "h2 0.3.25", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", @@ -2127,8 +2131,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.2", - "http 1.0.0", + "h2 0.4.3", + "http 1.1.0", "http-body 1.0.0", "httparse", "httpdate", @@ -2146,7 +2150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "rustls 0.21.10", "tokio", @@ -2166,6 +2170,22 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -2175,7 +2195,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "hyper 1.2.0", "pin-project-lite", @@ -2251,7 +2271,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.5", + "regex-automata 0.4.6", "same-file", "walkdir", "winapi-util", @@ -2295,9 +2315,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.3" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2311,7 +2331,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -2326,9 +2346,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.35.1" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c985c1bef99cf13c58fade470483d81a2bfe846ebde60ed28cc2dddec2df9e2" +checksum = "3eab73f58e59ca6526037208f0e98851159ec1633cf17b6cd2e1f2c3fd5d53cc" dependencies = [ "console", "lazy_static", @@ -2338,7 +2358,6 @@ dependencies = [ "regex", "serde", "similar", - "yaml-rust", ] [[package]] @@ -2387,24 +2406,33 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonwebtoken" -version = "9.2.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ "base64 0.21.7", "js-sys", @@ -2426,12 +2454,12 @@ dependencies = [ [[package]] name = "lettre" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357ff5edb6d8326473a64c82cf41ddf78ab116f89668c50c4fac1b321e5e80f4" +checksum = "8305b122b8ccc64e437b0de101851f9f00aade5886246e85f341c1d9a15a91b7" dependencies = [ "async-trait", - "base64 0.21.7", + "base64 0.22.0", "chumsky", "email-encoding", "email_address", @@ -2445,11 +2473,11 @@ dependencies = [ "nom", "percent-encoding", "quoted_printable", - "rustls 0.22.2", - "rustls-pemfile 2.1.0", + "rustls 0.23.4", + "rustls-pemfile 2.1.1", "socket2", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "url", "webpki-roots 0.26.1", ] @@ -2472,7 +2500,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", "redox_syscall", ] @@ -2508,9 +2536,9 @@ checksum = "c33f149bc6cef41a9f24ad43ece20c87e0617fc88affa01d95850eb68210daac" [[package]] name = "librqbit-core" -version = "3.5.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d055eac3cd062d4b7728feccb45495682b834305bf234f275440b32814276a65" +checksum = "09c1f868cbfa5df2721d1601befbf4aa69bfd80db725d4c331cd4783ad0f0da7" dependencies = [ "anyhow", "directories", @@ -2550,12 +2578,12 @@ dependencies = [ [[package]] name = "lightningcss" -version = "1.0.0-alpha.54" +version = "1.0.0-alpha.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d306844e5af1753490c420c0d6ae3d814b00725092d106332762827ca8f0fe" +checksum = "3bd5bed3814fb631bfc1e24c2be6f7e86a9837c660909acab79a38374dcb8798" dependencies = [ - "ahash 0.8.9", - "bitflags 2.4.2", + "ahash 0.8.11", + "bitflags 2.5.0", "const-str", "cssparser", "cssparser-color", @@ -2597,9 +2625,9 @@ dependencies = [ [[package]] name = "loco-rs" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629f92003dc023b90671f90869c5170145597ac0ca94296a6dacde85758fc566" +checksum = "4b1886d2d20ef60c490f02e9ac528876a322b025367ed70dbbe7e3f973ecfc49" dependencies = [ "argon2", "async-trait", @@ -2645,14 +2673,14 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", - "validator", + "validator 0.16.1", ] [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "maplit" @@ -2699,9 +2727,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mime" @@ -2736,9 +2764,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", @@ -2776,17 +2804,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.4.2", - "cfg-if", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -2892,9 +2909,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" +checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", "bytes", @@ -2919,9 +2936,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opendal" -version = "0.45.0" +version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3350be0d4ba326017ce22c98a9e94d21b069160fcd95bbe6c2555dac4e93c47a" +checksum = "52c17c077f23fa2d2c25d9d22af98baa43b8bbe2ef0de80cf66339aa70401467" dependencies = [ "anyhow", "async-trait", @@ -2932,14 +2949,14 @@ dependencies = [ "flagset", "futures", "getrandom", - "http 0.2.11", + "http 0.2.12", "log", "md-5", "once_cell", "percent-encoding", - "quick-xml 0.30.0", + "quick-xml 0.31.0", "reqsign", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "sha2", @@ -2953,7 +2970,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -2970,7 +2987,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -3052,11 +3069,11 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -3086,7 +3103,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05d74befe2d076330d9a58bf9ca2da424568724ab278adf15fb5718253133887" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cssparser", "fxhash", "log", @@ -3202,9 +3219,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" +checksum = "56f8023d0fb78c8e03784ea1c7f3fa36e68a723138990b8d5a47d916b651e7a8" dependencies = [ "memchr", "thiserror", @@ -3213,9 +3230,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e1288dbd7786462961e69bfd4df7848c1e37e8b74303dbdab82c3a9cdd2809" +checksum = "b0d24f72393fd16ab6ac5738bc33cdb6a9aa73f8b902e8fe29cf4e67d7dd1026" dependencies = [ "pest", "pest_generator", @@ -3223,22 +3240,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" +checksum = "fdc17e2a6c7d0a492f0158d7a4bd66cc17280308bbaff78d5bef566dca35ab80" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] name = "pest_meta" -version = "2.7.7" +version = "2.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0934d6907f148c22a3acbda520c7eed243ad7487a30f51f6ce52b58b7077a8a" +checksum = "934cd7631c050f4674352a6e835d5f6711ffbfb9345c2fc0107155ac495ae293" dependencies = [ "once_cell", "pest", @@ -3314,7 +3331,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -3337,22 +3354,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -3474,9 +3491,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -3516,7 +3533,7 @@ version = "0.4.1" source = "git+https://github.com/George-Miao/qbit.git?rev=ad5af6a#ad5af6a55b93b2c91b17d12d1b2ce54537df2355" dependencies = [ "mod_use", - "reqwest", + "reqwest 0.11.27", "serde", "serde-value", "serde_json", @@ -3537,7 +3554,6 @@ checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "encoding_rs", "memchr", - "serde", ] [[package]] @@ -3614,9 +3630,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -3646,7 +3662,6 @@ dependencies = [ "fancy-regex", "futures", "html-escape", - "include_dir", "insta", "itertools 0.12.1", "lazy_static", @@ -3659,7 +3674,7 @@ dependencies = [ "qbit-rs", "quirks_path", "regex", - "reqwest", + "reqwest 0.12.2", "rss", "rstest", "sea-orm", @@ -3677,7 +3692,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", - "validator", + "validator 0.17.0", "weak-table", ] @@ -3723,14 +3738,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -3744,13 +3759,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -3761,9 +3776,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "relative-path" @@ -3795,14 +3810,14 @@ dependencies = [ "hex 0.4.3", "hmac", "home", - "http 0.2.11", + "http 0.2.12", "jsonwebtoken", "log", "once_cell", "percent-encoding", "quick-xml 0.31.0", "rand", - "reqwest", + "reqwest 0.11.27", "rsa", "rust-ini", "serde", @@ -3839,21 +3854,21 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.24", - "http 0.2.11", + "h2 0.3.25", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "hyper-rustls", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -3868,7 +3883,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -3883,6 +3898,48 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "reserve-port" version = "2.0.1" @@ -3946,7 +4003,7 @@ dependencies = [ "cruet", "fs-err", "glob", - "heck", + "heck 0.4.1", "regex", "serde", "serde_json", @@ -4014,7 +4071,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.50", + "syn 2.0.55", "unicode-ident", ] @@ -4037,7 +4094,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 0.2.11", + "http 0.2.12", "mime", "mime_guess", "rand", @@ -4046,9 +4103,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.34.3" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39449a79f45e8da28c57c341891b69a183044b29518bb8f86dbac9df60bb7df" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" dependencies = [ "arrayvec", "borsh", @@ -4077,11 +4134,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -4102,11 +4159,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.2" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +checksum = "8c4d6d8ad9f2492485e13453acbb291dd08f64441b6609c491f1c2cd2c6b4fe1" dependencies = [ "log", + "once_cell", "ring", "rustls-pki-types", "rustls-webpki 0.102.2", @@ -4137,9 +4195,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ "base64 0.21.7", "rustls-pki-types", @@ -4147,9 +4205,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048a63e5b3ac996d78d402940b5fa47973d2d080c6c6fffa1d0f19c4445310b7" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" [[package]] name = "rustls-webpki" @@ -4189,7 +4247,7 @@ dependencies = [ "chrono", "cron_clock", "gethostname", - "heck", + "heck 0.4.1", "hex 0.4.3", "num_cpus", "rand", @@ -4270,18 +4328,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd3534a9978d0aa7edd2808dc1f8f31c4d0ecd31ddf71d997b3c98e9f3c9114" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] name = "sea-orm" -version = "1.0.0-rc.1" +version = "1.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b1218f3b8f95076954bde2e2a33852f873a91e5bbd49402f8fc2bb755c0ed3" +checksum = "9d5a485429878b8f6eb9ec1c99e91bf4e9055d445be71218faed5847c2e40dba" dependencies = [ "async-stream", "async-trait", @@ -4307,9 +4365,9 @@ dependencies = [ [[package]] name = "sea-orm-cli" -version = "1.0.0-rc.1" +version = "1.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096c0ffb155732ab98827d6a119c28166073933cb60e7662cc87f8b8af71ee31" +checksum = "c7fc1c75b5075c7bc2aa41518731e93408f3a158dfb8714e747430734934a901" dependencies = [ "chrono", "clap", @@ -4324,23 +4382,23 @@ dependencies = [ [[package]] name = "sea-orm-macros" -version = "1.0.0-rc.1" +version = "1.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be16d30795cc707c355d1c0ba704db085d5bd507a858509a0d784189b8fe31b" +checksum = "3f303a2b29c6d92a669652fa7776e6590df7118e05d4f6bf31d48127b06cb21a" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "sea-bae", - "syn 2.0.50", + "syn 2.0.55", "unicode-ident", ] [[package]] name = "sea-orm-migration" -version = "1.0.0-rc.1" +version = "1.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4daac7104c6e919c4ae2da226b1ccb5d9b8c67046d85a597e20dee815d20319" +checksum = "8e8b13ba1f4308423946530237d5b46f8bef85aec2463686c4eb1e45d87ce9ef" dependencies = [ "async-trait", "clap", @@ -4393,18 +4451,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a82fcb49253abcb45cdcb2adf92956060ec0928635eb21b4f7a6d8f25ab0bc" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", "thiserror", ] [[package]] name = "sea-schema" -version = "0.15.0-rc.2" +version = "0.15.0-rc.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a03b47e6cf73c34d83fff934c169732530f6f3b0dd072725804961959b7d1df" +checksum = "20ed3975863149bdc9dcc19007d55c16255e9805ab85c1e9792d4bb5d2a694d6" dependencies = [ "futures", "sea-query", @@ -4417,7 +4475,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -4488,14 +4546,14 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -4504,9 +4562,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -4530,7 +4588,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -4601,16 +4659,16 @@ dependencies = [ "darling 0.20.8", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] name = "serde_yaml" -version = "0.9.32" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.3", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -4619,9 +4677,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +checksum = "953ad9342b3aaca7cb43c45c097dd008d4907070394bd0751a0aa8817e5a018d" dependencies = [ "dashmap", "futures", @@ -4633,13 +4691,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -4823,9 +4881,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "smawk" @@ -4849,7 +4907,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -4903,9 +4961,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" dependencies = [ "sqlx-core", "sqlx-macros", @@ -4916,11 +4974,11 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ - "ahash 0.8.9", + "ahash 0.8.11", "atoi", "bigdecimal", "byteorder", @@ -4928,7 +4986,6 @@ dependencies = [ "chrono", "crc", "crossbeam-queue", - "dotenvy", "either", "event-listener", "futures-channel", @@ -4938,7 +4995,7 @@ dependencies = [ "futures-util", "hashlink", "hex 0.4.3", - "indexmap 2.2.3", + "indexmap 2.2.6", "log", "memchr", "once_cell", @@ -4964,9 +5021,9 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" dependencies = [ "proc-macro2", "quote", @@ -4977,14 +5034,13 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" dependencies = [ - "atomic-write-file", "dotenvy", "either", - "heck", + "heck 0.4.1", "hex 0.4.3", "once_cell", "proc-macro2", @@ -5004,14 +5060,14 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" dependencies = [ "atoi", "base64 0.21.7", "bigdecimal", - "bitflags 2.4.2", + "bitflags 2.5.0", "byteorder", "bytes", "chrono", @@ -5051,14 +5107,14 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" dependencies = [ "atoi", "base64 0.21.7", "bigdecimal", - "bitflags 2.4.2", + "bitflags 2.5.0", "byteorder", "chrono", "crc", @@ -5082,7 +5138,6 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha1", "sha2", "smallvec", "sqlx-core", @@ -5096,9 +5151,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" dependencies = [ "atoi", "chrono", @@ -5164,9 +5219,9 @@ checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" [[package]] name = "subtle" @@ -5187,9 +5242,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.50" +version = "2.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" dependencies = [ "proc-macro2", "quote", @@ -5205,7 +5260,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -5214,6 +5269,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384595c11a4e2969895cad5a8c4029115f5ab956a9e5ef4de79d11a426e5f20c" + [[package]] name = "system-configuration" version = "0.5.1" @@ -5243,9 +5304,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", @@ -5305,9 +5366,9 @@ dependencies = [ [[package]] name = "testcontainers-modules" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0334776e1e8ee7c504a922c5236daf865ffe413aa630d84ae91dcce0b10bc3" +checksum = "204d1c7516bfdc8a01bb85d3e30145e5bbeb2351812e5e8aa6971769109b45b5" dependencies = [ "testcontainers", ] @@ -5325,22 +5386,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -5441,7 +5502,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -5466,20 +5527,20 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.22.2", + "rustls 0.23.4", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -5524,7 +5585,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.3", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -5552,11 +5613,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "async-compression", - "bitflags 2.4.2", + "bitflags 2.5.0", "bytes", "futures-core", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "http-range-header", @@ -5604,7 +5665,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -5804,9 +5865,9 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -5846,9 +5907,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom", "serde", @@ -5870,6 +5931,21 @@ dependencies = [ "validator_derive", ] +[[package]] +name = "validator" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da339118f018cc70ebf01fafc103360528aad53717e4bf311db929cb01cb9345" +dependencies = [ + "idna 0.5.0", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", +] + [[package]] name = "validator_derive" version = "0.16.0" @@ -5922,9 +5998,9 @@ checksum = "65dd7eed29412da847b0f78bcec0ac98588165988a8cfe41d4ea1d429f8ccfff" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -5946,10 +6022,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasm-bindgen" -version = "0.2.91" +name = "wasite" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -5957,24 +6039,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -5984,9 +6066,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5994,22 +6076,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" @@ -6032,9 +6114,9 @@ checksum = "323f4da9523e9a669e1eaf9c6e763892769b1d38c623913647bfdc1532fe4549" [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -6057,9 +6139,13 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall", + "wasite", +] [[package]] name = "winapi" @@ -6098,7 +6184,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -6116,7 +6202,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -6136,17 +6222,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.3", - "windows_aarch64_msvc 0.52.3", - "windows_i686_gnu 0.52.3", - "windows_i686_msvc 0.52.3", - "windows_x86_64_gnu 0.52.3", - "windows_x86_64_gnullvm 0.52.3", - "windows_x86_64_msvc 0.52.3", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -6157,9 +6243,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -6169,9 +6255,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -6181,9 +6267,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -6193,9 +6279,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -6205,9 +6291,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -6217,9 +6303,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -6229,9 +6315,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" @@ -6267,15 +6353,6 @@ dependencies = [ "tap", ] -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yansi" version = "0.5.1" @@ -6299,7 +6376,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.50", + "syn 2.0.55", ] [[package]] @@ -6310,27 +6387,27 @@ checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 9a78dcc..ee230a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,9 @@ cargo-features = ["codegen-backend"] [workspace] -members = ["crates/quirks_path", "crates/recorder"] +members = [ + "crates/quirks_path", + "crates/recorder" +] resolver = "2" [profile.dev] diff --git a/config/recorder.development.yaml b/config/recorder.development.yaml index 204a422..7f220c0 100644 --- a/config/recorder.development.yaml +++ b/config/recorder.development.yaml @@ -67,22 +67,6 @@ workers: # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. mode: BackgroundQueue -# Mailer Configuration. -mailer: - # SMTP mailer configuration. - smtp: - # Enable/Disable smtp mailer. - enable: true - # SMTP server host. e.x localhost, smtp.gmail.com - host: '{{ get_env(name="MAILER_HOST", default="localhost") }}' - # SMTP server port - port: 1025 - # Use secure connection (SSL/TLS). - secure: false - # auth: - # user: - # password: - # Database Configuration database: # Database connection URI @@ -104,10 +88,3 @@ database: # Recreating schema when application loaded. This is a dangerous operation, make sure that you using this flag only on dev environments or test mode dangerously_recreate: false -# Redis Configuration -redis: - # Redis connection URI - uri: '{{ get_env(name="REDIS_URL", default="redis://127.0.0.1:6379") }}' - # Dangerously flush all data in Redis on startup. dangerous operation, make sure that you using this flag only on dev environments or test mode - dangerously_flush: false - diff --git a/config/recorder.test.yaml b/config/recorder.test.yaml index b963126..cd04964 100644 --- a/config/recorder.test.yaml +++ b/config/recorder.test.yaml @@ -62,27 +62,10 @@ workers: # - BackgroundAsync - Workers operate asynchronously in the background, processing tasks with async capabilities. mode: ForegroundBlocking -# Mailer Configuration. -mailer: - # SMTP mailer configuration. - smtp: - # Enable/Disable smtp mailer. - enable: true - # SMTP server host. e.x localhost, smtp.gmail.com - host: localhost - # SMTP server port - port: 1025 - # Use secure connection (SSL/TLS). - secure: false - # auth: - # user: - # password: - stub: true - # Database Configuration database: # Database connection URI - uri: {{get_env(name="DATABASE_URL", default="postgres://loco:loco@localhost:5432/recorder_test")}} + uri: {{get_env(name="DATABASE_URL", default="postgres://loco:loco@localhost:5432/recorder_test")}} # When enabled, the sql query will be logged. enable_logging: false # Set the timeout duration when acquiring a connection. @@ -107,12 +90,3 @@ redis: # Dangerously flush all data in Redis on startup. dangerous operation, make sure that you using this flag only on dev environments or test mode dangerously_flush: false -# Authentication Configuration -auth: - # JWT authentication - jwt: - # Secret key for token generation and verification - secret: ZknFYqXpnDgaWcKJZ5J5 - # Token expiration time in seconds - expiration: 604800 # 7 days - diff --git a/crates/recorder/Cargo.toml b/crates/recorder/Cargo.toml index 81ff2d1..d90c1bc 100644 --- a/crates/recorder/Cargo.toml +++ b/crates/recorder/Cargo.toml @@ -18,30 +18,28 @@ default = [] testcontainers = [] [dependencies] -loco-rs = { version = "0.3.1" } +loco-rs = { version = "0.3.2" } serde = { version = "1", features = ["derive"] } serde_json = "1" eyre = "0.6" -tokio = { version = "1.33.0", default-features = false } -async-trait = "0.1.74" +tokio = { version = "1.36.0", default-features = false } +async-trait = "0.1.79" tracing = "0.1.40" chrono = "0.4" -validator = { version = "0.16" } -sea-orm = { version = "1.0.0-rc.1", features = [ +validator = { version = "0.17" } +sea-orm = { version = "1.0.0-rc.3", features = [ "sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", ] } - -axum = "0.7.1" -include_dir = "0.7" -uuid = { version = "1.6.0", features = ["v4"] } +axum = "0.7.5" +uuid = { version = "1.8.0", features = ["v4"] } tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json"] } -sea-orm-migration = { version = "1.0.0-rc.1", features = [ +sea-orm-migration = { version = "1.0.0-rc.3", features = [ "runtime-tokio-rustls", ] } -reqwest = "0.11.24" +reqwest = { version = "0.12.2", features = ["json"] } thiserror = "1.0.57" rss = "2.0.7" bytes = "1.5.0" @@ -56,19 +54,19 @@ maplit = "1.0.2" tl = { version = "0.7.8", features = ["simd"] } lightningcss = "1.0.0-alpha.54" html-escape = "0.2.13" -opendal = "0.45.0" -librqbit-core = "3.5.0" +opendal = "0.45.1" +librqbit-core = "3.6.1" quirks_path = { path = "../quirks_path" } tokio-utils = "0.1.2" -weak-table = "0.3.2" oxilangtag = { version = "0.1.5", features = ["serde"] } dateparser = "0.2.1" dotenv = "0.15.0" +weak-table = "0.3.2" [dev-dependencies] -serial_test = "2.0.0" +serial_test = "3.0.0" rstest = "0.18.2" -loco-rs = { version = "0.3.1", features = ["testing"] } -insta = { version = "1.34.0", features = ["redactions", "yaml", "filters"] } +loco-rs = { version = "0.3.2", features = ["testing"] } +insta = { version = "1.3", features = ["redactions", "yaml", "filters"] } testcontainers = { version = "0.15.0" } -testcontainers-modules = { version = "0.3.5" } +testcontainers-modules = { version = "0.3.6" } diff --git a/crates/recorder/src/app.rs b/crates/recorder/src/app.rs index 13bdb14..8c462a8 100644 --- a/crates/recorder/src/app.rs +++ b/crates/recorder/src/app.rs @@ -2,7 +2,7 @@ use std::path::Path; use async_trait::async_trait; use loco_rs::{ - app::{AppContext, Hooks}, + app::{AppContext, Hooks, Initializer}, boot::{create_app, BootResult, StartMode}, controller::AppRoutes, db::truncate_table, @@ -14,7 +14,7 @@ use loco_rs::{ use sea_orm::DatabaseConnection; use crate::{ - controllers, migrations::Migrator, models::entities::subscribers, + controllers, migrations::Migrator, models::entities::subscribers, storage::AppDalInitializer, workers::subscription::SubscriptionWorker, }; @@ -60,4 +60,8 @@ impl Hooks for App { async fn seed(_db: &DatabaseConnection, _base: &Path) -> Result<()> { Ok(()) } + + async fn initializers(_ctx: &AppContext) -> Result>> { + Ok(vec![Box::new(AppDalInitializer)]) + } } diff --git a/crates/recorder/src/config/mod.rs b/crates/recorder/src/config/mod.rs index de881d8..02e79c8 100644 --- a/crates/recorder/src/config/mod.rs +++ b/crates/recorder/src/config/mod.rs @@ -4,7 +4,7 @@ use eyre::OptionExt; use itertools::Itertools; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -pub const DAL_CONF_KEY: &str = "dal"; +pub const DAL_CONF_KEY: &str = "storage"; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AppCustomConf { diff --git a/crates/recorder/src/downloaders/api_client.rs b/crates/recorder/src/downloaders/api_client.rs new file mode 100644 index 0000000..67e4769 --- /dev/null +++ b/crates/recorder/src/downloaders/api_client.rs @@ -0,0 +1,76 @@ +use axum::http::HeaderMap; +use bytes::Bytes; +use serde::de::DeserializeOwned; +use tokio_utils::RateLimiter; + +use crate::downloaders::defs::DEFAULT_USER_AGENT; + +pub struct ApiClient { + headers: HeaderMap, + rate_limiter: RateLimiter, + fetch_client: reqwest::Client, +} + +impl ApiClient { + pub fn new( + throttle_duration: std::time::Duration, + override_headers: Option, + ) -> eyre::Result { + Ok(Self { + headers: override_headers.unwrap_or_else(HeaderMap::new), + rate_limiter: RateLimiter::new(throttle_duration), + fetch_client: reqwest::Client::builder() + .user_agent(DEFAULT_USER_AGENT) + .build()?, + }) + } + + pub async fn fetch_json(&self, f: F) -> Result + where + F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, + R: DeserializeOwned, + { + self.rate_limiter + .throttle(|| async { + f(&self.fetch_client) + .headers(self.headers.clone()) + .send() + .await? + .json::() + .await + }) + .await + } + + pub async fn fetch_bytes(&self, f: F) -> Result + where + F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, + { + self.rate_limiter + .throttle(|| async { + f(&self.fetch_client) + .headers(self.headers.clone()) + .send() + .await? + .bytes() + .await + }) + .await + } + + pub async fn fetch_text(&self, f: F) -> Result + where + F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, + { + self.rate_limiter + .throttle(|| async { + f(&self.fetch_client) + .headers(self.headers.clone()) + .send() + .await? + .text() + .await + }) + .await + } +} diff --git a/crates/recorder/src/downloaders/defs.rs b/crates/recorder/src/downloaders/defs.rs index eb87f9c..1683a5b 100644 --- a/crates/recorder/src/downloaders/defs.rs +++ b/crates/recorder/src/downloaders/defs.rs @@ -1,23 +1,11 @@ use bytes::Bytes; -use itertools::Itertools; -use lazy_static::lazy_static; -use librqbit_core::{ - magnet::Magnet, - torrent_metainfo::{torrent_from_bytes, TorrentMetaV1Owned}, -}; pub use qbit_rs::model::{ Torrent as QbitTorrent, TorrentContent as QbitTorrentContent, TorrentFilter as QbitTorrentFilter, TorrentSource as QbitTorrentSource, }; -use regex::Regex; -use reqwest::{header::HeaderMap, IntoUrl}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tokio_utils::RateLimiter; -use url::Url; +use reqwest::IntoUrl; -use super::error::DownloaderError; - -async fn download_bytes(url: T) -> eyre::Result { +pub(crate) async fn download_bytes(url: T) -> eyre::Result { let request_client = reqwest::Client::builder() .user_agent(DEFAULT_USER_AGENT) .build()?; @@ -28,303 +16,3 @@ async fn download_bytes(url: T) -> eyre::Result { pub const BITTORRENT_MIME_TYPE: &str = "application/x-bittorrent"; pub const MAGNET_SCHEMA: &str = "magnet"; pub const DEFAULT_USER_AGENT: &str = "Wget/1.13.4 (linux-gnu)"; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TorrentFilter { - All, - Downloading, - Completed, - Paused, - Active, - Inactive, - Resumed, - Stalled, - StalledUploading, - StalledDownloading, - Errored, -} - -impl From for QbitTorrentFilter { - fn from(val: TorrentFilter) -> Self { - match val { - TorrentFilter::All => QbitTorrentFilter::All, - TorrentFilter::Downloading => QbitTorrentFilter::Downloading, - TorrentFilter::Completed => QbitTorrentFilter::Completed, - TorrentFilter::Paused => QbitTorrentFilter::Paused, - TorrentFilter::Active => QbitTorrentFilter::Active, - TorrentFilter::Inactive => QbitTorrentFilter::Inactive, - TorrentFilter::Resumed => QbitTorrentFilter::Resumed, - TorrentFilter::Stalled => QbitTorrentFilter::Stalled, - TorrentFilter::StalledUploading => QbitTorrentFilter::StalledUploading, - TorrentFilter::StalledDownloading => QbitTorrentFilter::StalledDownloading, - TorrentFilter::Errored => QbitTorrentFilter::Errored, - } - } -} - -lazy_static! { - static ref TORRENT_HASH_RE: Regex = Regex::new(r"[a-fA-F0-9]{40}").unwrap(); - static ref TORRENT_EXT_RE: Regex = Regex::new(r"\.torrent$").unwrap(); -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum TorrentSource { - MagnetUrl { - url: Url, - hash: String, - }, - TorrentUrl { - url: Url, - hash: String, - }, - TorrentFile { - torrent: Vec, - hash: String, - name: Option, - }, -} - -impl TorrentSource { - pub async fn parse(url: &str) -> eyre::Result { - let url = Url::parse(url)?; - let source = if url.scheme() == MAGNET_SCHEMA { - TorrentSource::from_magnet_url(url)? - } else if let Some(basename) = url - .clone() - .path_segments() - .and_then(|segments| segments.last()) - { - if let (Some(match_hash), true) = ( - TORRENT_HASH_RE.find(basename), - TORRENT_EXT_RE.is_match(basename), - ) { - TorrentSource::from_torrent_url(url, match_hash.as_str().to_string())? - } else { - let contents = download_bytes(url).await?; - TorrentSource::from_torrent_file(contents.to_vec(), Some(basename.to_string()))? - } - } else { - let contents = download_bytes(url).await?; - TorrentSource::from_torrent_file(contents.to_vec(), None)? - }; - Ok(source) - } - - pub fn from_torrent_file(file: Vec, name: Option) -> eyre::Result { - let torrent: TorrentMetaV1Owned = - torrent_from_bytes(&file).map_err(|_| DownloaderError::InvalidTorrentFileFormat)?; - let hash = torrent.info_hash.as_string(); - Ok(TorrentSource::TorrentFile { - torrent: file, - hash, - name, - }) - } - - pub fn from_magnet_url(url: Url) -> eyre::Result { - if url.scheme() != MAGNET_SCHEMA { - Err(DownloaderError::InvalidUrlSchema { - found: url.scheme().to_string(), - expected: MAGNET_SCHEMA.to_string(), - } - .into()) - } else { - let magnet = - Magnet::parse(url.as_str()).map_err(|_| DownloaderError::InvalidMagnetFormat { - url: url.as_str().to_string(), - })?; - let hash = magnet.info_hash.as_string(); - Ok(TorrentSource::MagnetUrl { url, hash }) - } - } - - pub fn from_torrent_url(url: Url, hash: String) -> eyre::Result { - Ok(TorrentSource::TorrentUrl { url, hash }) - } - - pub fn hash(&self) -> &str { - match self { - TorrentSource::MagnetUrl { hash, .. } => hash, - TorrentSource::TorrentUrl { hash, .. } => hash, - TorrentSource::TorrentFile { hash, .. } => hash, - } - } -} - -impl From for QbitTorrentSource { - fn from(value: TorrentSource) -> Self { - match value { - TorrentSource::MagnetUrl { url, .. } => QbitTorrentSource::Urls { - urls: qbit_rs::model::Sep::from([url]), - }, - TorrentSource::TorrentUrl { url, .. } => QbitTorrentSource::Urls { - urls: qbit_rs::model::Sep::from([url]), - }, - TorrentSource::TorrentFile { - torrent: torrents, .. - } => QbitTorrentSource::TorrentFiles { torrents }, - } - } -} - -pub trait TorrentContent { - fn get_name(&self) -> &str; - - fn get_all_size(&self) -> u64; - - fn get_progress(&self) -> f64; - - fn get_curr_size(&self) -> u64; -} - -impl TorrentContent for QbitTorrentContent { - fn get_name(&self) -> &str { - self.name.as_str() - } - - fn get_all_size(&self) -> u64 { - self.size - } - - fn get_progress(&self) -> f64 { - self.progress - } - - fn get_curr_size(&self) -> u64 { - u64::clamp( - f64::round(self.get_all_size() as f64 * self.get_progress()) as u64, - 0, - self.get_all_size(), - ) - } -} - -#[derive(Debug, Clone)] -pub enum Torrent { - Qbit { - torrent: QbitTorrent, - contents: Vec, - }, -} - -impl Torrent { - pub fn iter_files(&self) -> impl Iterator { - match self { - Torrent::Qbit { contents, .. } => { - contents.iter().map(|item| item as &dyn TorrentContent) - } - } - } - - pub fn get_name(&self) -> Option<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.name.as_deref(), - } - } - - pub fn get_hash(&self) -> Option<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.hash.as_deref(), - } - } - - pub fn get_save_path(&self) -> Option<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.save_path.as_deref(), - } - } - - pub fn get_content_path(&self) -> Option<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.content_path.as_deref(), - } - } - - pub fn get_tags(&self) -> Vec<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.tags.as_deref().map_or_else(Vec::new, |s| { - s.split(',') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .collect_vec() - }), - } - } - - pub fn get_category(&self) -> Option<&str> { - match self { - Torrent::Qbit { torrent, .. } => torrent.category.as_deref(), - } - } -} - -pub struct ApiClient { - headers: HeaderMap, - rate_limiter: RateLimiter, - fetch_client: reqwest::Client, -} - -impl ApiClient { - pub fn new( - throttle_duration: std::time::Duration, - override_headers: Option, - ) -> eyre::Result { - Ok(Self { - headers: override_headers.unwrap_or_else(HeaderMap::new), - rate_limiter: RateLimiter::new(throttle_duration), - fetch_client: reqwest::Client::builder() - .user_agent(DEFAULT_USER_AGENT) - .build()?, - }) - } - - pub async fn fetch_json(&self, f: F) -> Result - where - F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, - R: DeserializeOwned, - { - self.rate_limiter - .throttle(|| async { - f(&self.fetch_client) - .headers(self.headers.clone()) - .send() - .await? - .json::() - .await - }) - .await - } - - pub async fn fetch_bytes(&self, f: F) -> Result - where - F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, - { - self.rate_limiter - .throttle(|| async { - f(&self.fetch_client) - .headers(self.headers.clone()) - .send() - .await? - .bytes() - .await - }) - .await - } - - pub async fn fetch_text(&self, f: F) -> Result - where - F: FnOnce(&reqwest::Client) -> reqwest::RequestBuilder, - { - self.rate_limiter - .throttle(|| async { - f(&self.fetch_client) - .headers(self.headers.clone()) - .send() - .await? - .text() - .await - }) - .await - } -} diff --git a/crates/recorder/src/downloaders/mod.rs b/crates/recorder/src/downloaders/mod.rs index a229823..620d4a8 100644 --- a/crates/recorder/src/downloaders/mod.rs +++ b/crates/recorder/src/downloaders/mod.rs @@ -1,4 +1,7 @@ +pub mod api_client; pub mod defs; pub mod error; pub mod qbitorrent; -pub mod torrent_downloader; +pub mod torrent; + +pub use api_client::ApiClient; diff --git a/crates/recorder/src/downloaders/qbitorrent.rs b/crates/recorder/src/downloaders/qbitorrent.rs index 4bcc966..d4a6def 100644 --- a/crates/recorder/src/downloaders/qbitorrent.rs +++ b/crates/recorder/src/downloaders/qbitorrent.rs @@ -17,13 +17,12 @@ use quirks_path::{path_equals_as_file_url, Path, PathBuf}; use tokio::time::sleep; use url::Url; -use super::{ - defs::{Torrent, TorrentFilter, TorrentSource}, - error::DownloaderError, - torrent_downloader::TorrentDownloader, -}; +use super::error::DownloaderError; use crate::{ - downloaders::defs::{QbitTorrent, QbitTorrentContent, TorrentContent}, + downloaders::{ + defs::{QbitTorrent, QbitTorrentContent}, + torrent::{Torrent, TorrentContent, TorrentDownloader, TorrentFilter, TorrentSource}, + }, models::{entities::downloaders, prelude::DownloaderCategory}, }; @@ -414,7 +413,7 @@ impl Debug for QBittorrentDownloader { } #[cfg(test)] -pub mod tests { +pub(crate) mod tests { use itertools::Itertools; use super::*; diff --git a/crates/recorder/src/downloaders/torrent.rs b/crates/recorder/src/downloaders/torrent.rs new file mode 100644 index 0000000..cb01c59 --- /dev/null +++ b/crates/recorder/src/downloaders/torrent.rs @@ -0,0 +1,341 @@ +use eyre::OptionExt; +use itertools::Itertools; +use lazy_static::lazy_static; +use librqbit_core::{ + magnet::Magnet, + torrent_metainfo::{torrent_from_bytes, TorrentMetaV1Owned}, +}; +use quirks_path::{Path, PathBuf}; +use regex::Regex; +use sea_orm::{ActiveModelTrait, ActiveValue, DatabaseConnection, IntoActiveModel}; +use serde::{Deserialize, Serialize}; +use url::Url; + +use super::{ + defs::{ + download_bytes, QbitTorrent, QbitTorrentContent, QbitTorrentFilter, QbitTorrentSource, + MAGNET_SCHEMA, + }, + error::DownloaderError, + qbitorrent::QBittorrentDownloader, +}; +use crate::{ + models::{bangumi, downloaders, downloaders::DownloaderCategory, downloads}, + path::torrent_path::gen_bangumi_sub_path, +}; + +lazy_static! { + static ref TORRENT_HASH_RE: Regex = Regex::new(r"[a-fA-F0-9]{40}").unwrap(); + static ref TORRENT_EXT_RE: Regex = Regex::new(r"\.torrent$").unwrap(); +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TorrentFilter { + All, + Downloading, + Completed, + Paused, + Active, + Inactive, + Resumed, + Stalled, + StalledUploading, + StalledDownloading, + Errored, +} + +impl From for QbitTorrentFilter { + fn from(val: TorrentFilter) -> Self { + match val { + TorrentFilter::All => QbitTorrentFilter::All, + TorrentFilter::Downloading => QbitTorrentFilter::Downloading, + TorrentFilter::Completed => QbitTorrentFilter::Completed, + TorrentFilter::Paused => QbitTorrentFilter::Paused, + TorrentFilter::Active => QbitTorrentFilter::Active, + TorrentFilter::Inactive => QbitTorrentFilter::Inactive, + TorrentFilter::Resumed => QbitTorrentFilter::Resumed, + TorrentFilter::Stalled => QbitTorrentFilter::Stalled, + TorrentFilter::StalledUploading => QbitTorrentFilter::StalledUploading, + TorrentFilter::StalledDownloading => QbitTorrentFilter::StalledDownloading, + TorrentFilter::Errored => QbitTorrentFilter::Errored, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TorrentSource { + MagnetUrl { + url: Url, + hash: String, + }, + TorrentUrl { + url: Url, + hash: String, + }, + TorrentFile { + torrent: Vec, + hash: String, + name: Option, + }, +} + +impl TorrentSource { + pub async fn parse(url: &str) -> eyre::Result { + let url = Url::parse(url)?; + let source = if url.scheme() == MAGNET_SCHEMA { + TorrentSource::from_magnet_url(url)? + } else if let Some(basename) = url + .clone() + .path_segments() + .and_then(|segments| segments.last()) + { + if let (Some(match_hash), true) = ( + TORRENT_HASH_RE.find(basename), + TORRENT_EXT_RE.is_match(basename), + ) { + TorrentSource::from_torrent_url(url, match_hash.as_str().to_string())? + } else { + let contents = download_bytes(url).await?; + TorrentSource::from_torrent_file(contents.to_vec(), Some(basename.to_string()))? + } + } else { + let contents = download_bytes(url).await?; + TorrentSource::from_torrent_file(contents.to_vec(), None)? + }; + Ok(source) + } + + pub fn from_torrent_file(file: Vec, name: Option) -> eyre::Result { + let torrent: TorrentMetaV1Owned = + torrent_from_bytes(&file).map_err(|_| DownloaderError::InvalidTorrentFileFormat)?; + let hash = torrent.info_hash.as_string(); + Ok(TorrentSource::TorrentFile { + torrent: file, + hash, + name, + }) + } + + pub fn from_magnet_url(url: Url) -> eyre::Result { + if url.scheme() != MAGNET_SCHEMA { + Err(DownloaderError::InvalidUrlSchema { + found: url.scheme().to_string(), + expected: MAGNET_SCHEMA.to_string(), + } + .into()) + } else { + let magnet = + Magnet::parse(url.as_str()).map_err(|_| DownloaderError::InvalidMagnetFormat { + url: url.as_str().to_string(), + })?; + let hash = magnet + .as_id20() + .ok_or_eyre("no info hash found")? + .as_string(); + Ok(TorrentSource::MagnetUrl { url, hash }) + } + } + + pub fn from_torrent_url(url: Url, hash: String) -> eyre::Result { + Ok(TorrentSource::TorrentUrl { url, hash }) + } + + pub fn hash(&self) -> &str { + match self { + TorrentSource::MagnetUrl { hash, .. } => hash, + TorrentSource::TorrentUrl { hash, .. } => hash, + TorrentSource::TorrentFile { hash, .. } => hash, + } + } +} + +impl From for QbitTorrentSource { + fn from(value: TorrentSource) -> Self { + match value { + TorrentSource::MagnetUrl { url, .. } => QbitTorrentSource::Urls { + urls: qbit_rs::model::Sep::from([url]), + }, + TorrentSource::TorrentUrl { url, .. } => QbitTorrentSource::Urls { + urls: qbit_rs::model::Sep::from([url]), + }, + TorrentSource::TorrentFile { + torrent: torrents, .. + } => QbitTorrentSource::TorrentFiles { torrents }, + } + } +} + +pub trait TorrentContent { + fn get_name(&self) -> &str; + + fn get_all_size(&self) -> u64; + + fn get_progress(&self) -> f64; + + fn get_curr_size(&self) -> u64; +} + +impl TorrentContent for QbitTorrentContent { + fn get_name(&self) -> &str { + self.name.as_str() + } + + fn get_all_size(&self) -> u64 { + self.size + } + + fn get_progress(&self) -> f64 { + self.progress + } + + fn get_curr_size(&self) -> u64 { + u64::clamp( + f64::round(self.get_all_size() as f64 * self.get_progress()) as u64, + 0, + self.get_all_size(), + ) + } +} + +#[derive(Debug, Clone)] +pub enum Torrent { + Qbit { + torrent: QbitTorrent, + contents: Vec, + }, +} + +impl Torrent { + pub fn iter_files(&self) -> impl Iterator { + match self { + Torrent::Qbit { contents, .. } => { + contents.iter().map(|item| item as &dyn TorrentContent) + } + } + } + + pub fn get_name(&self) -> Option<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.name.as_deref(), + } + } + + pub fn get_hash(&self) -> Option<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.hash.as_deref(), + } + } + + pub fn get_save_path(&self) -> Option<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.save_path.as_deref(), + } + } + + pub fn get_content_path(&self) -> Option<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.content_path.as_deref(), + } + } + + pub fn get_tags(&self) -> Vec<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.tags.as_deref().map_or_else(Vec::new, |s| { + s.split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect_vec() + }), + } + } + + pub fn get_category(&self) -> Option<&str> { + match self { + Torrent::Qbit { torrent, .. } => torrent.category.as_deref(), + } + } +} + +#[async_trait::async_trait] +pub trait TorrentDownloader { + async fn get_torrents_info( + &self, + status_filter: TorrentFilter, + category: Option, + tag: Option, + ) -> eyre::Result>; + + async fn add_torrents( + &self, + source: TorrentSource, + save_path: String, + category: Option<&str>, + ) -> eyre::Result<()>; + + async fn delete_torrents(&self, hashes: Vec) -> eyre::Result<()>; + + async fn rename_torrent_file( + &self, + hash: &str, + old_path: &str, + new_path: &str, + ) -> eyre::Result<()>; + + async fn move_torrents(&self, hashes: Vec, new_path: &str) -> eyre::Result<()>; + + async fn get_torrent_path(&self, hashes: String) -> eyre::Result>; + + async fn check_connection(&self) -> eyre::Result<()>; + + async fn set_torrents_category(&self, hashes: Vec, category: &str) -> eyre::Result<()>; + + async fn add_torrent_tags(&self, hashes: Vec, tags: Vec) -> eyre::Result<()>; + + async fn add_category(&self, category: &str) -> eyre::Result<()>; + + fn get_save_path(&self, sub_path: &Path) -> PathBuf; + + async fn add_downloads_for_bangumi<'a, 'b>( + &self, + db: &'a DatabaseConnection, + downloads: &[&downloads::Model], + mut bangumi: bangumi::Model, + ) -> eyre::Result { + if bangumi.save_path.is_none() { + let gen_sub_path = gen_bangumi_sub_path(&bangumi); + let mut bangumi_active = bangumi.into_active_model(); + bangumi_active.save_path = ActiveValue::Set(Some(gen_sub_path.to_string())); + bangumi = bangumi_active.update(db).await?; + } + + let sub_path = bangumi + .save_path + .as_ref() + .unwrap_or_else(|| unreachable!("must have a sub path")); + + let mut torrent_urls = vec![]; + for m in downloads.iter() { + torrent_urls.push(Url::parse(&m.url as &str)?); + } + + // make sequence to prevent too fast to be banned + for d in downloads.iter() { + let source = TorrentSource::parse(&d.url).await?; + self.add_torrents(source, sub_path.clone(), Some("bangumi")) + .await?; + } + + Ok(bangumi) + } +} + +pub async fn build_torrent_downloader_from_downloader_model( + model: downloaders::Model, +) -> eyre::Result> { + Ok(Box::new(match &model.category { + DownloaderCategory::QBittorrent => { + QBittorrentDownloader::from_downloader_model(model).await? + } + })) +} diff --git a/crates/recorder/src/downloaders/torrent_downloader.rs b/crates/recorder/src/downloaders/torrent_downloader.rs deleted file mode 100644 index 55a1a3b..0000000 --- a/crates/recorder/src/downloaders/torrent_downloader.rs +++ /dev/null @@ -1,96 +0,0 @@ -use downloaders::DownloaderCategory; -use quirks_path::{Path, PathBuf}; -use sea_orm::{ActiveModelTrait, ActiveValue, DatabaseConnection, IntoActiveModel}; -use url::Url; - -use super::{ - defs::{Torrent, TorrentFilter, TorrentSource}, - qbitorrent::QBittorrentDownloader, -}; -use crate::{ - models::{bangumi, downloaders, downloads}, - path::torrent_path::gen_bangumi_sub_path, -}; - -#[async_trait::async_trait] -pub trait TorrentDownloader { - async fn get_torrents_info( - &self, - status_filter: TorrentFilter, - category: Option, - tag: Option, - ) -> eyre::Result>; - - async fn add_torrents( - &self, - source: TorrentSource, - save_path: String, - category: Option<&str>, - ) -> eyre::Result<()>; - - async fn delete_torrents(&self, hashes: Vec) -> eyre::Result<()>; - - async fn rename_torrent_file( - &self, - hash: &str, - old_path: &str, - new_path: &str, - ) -> eyre::Result<()>; - - async fn move_torrents(&self, hashes: Vec, new_path: &str) -> eyre::Result<()>; - - async fn get_torrent_path(&self, hashes: String) -> eyre::Result>; - - async fn check_connection(&self) -> eyre::Result<()>; - - async fn set_torrents_category(&self, hashes: Vec, category: &str) -> eyre::Result<()>; - - async fn add_torrent_tags(&self, hashes: Vec, tags: Vec) -> eyre::Result<()>; - - async fn add_category(&self, category: &str) -> eyre::Result<()>; - - fn get_save_path(&self, sub_path: &Path) -> PathBuf; - - async fn add_downloads_for_bangumi<'a, 'b>( - &self, - db: &'a DatabaseConnection, - downloads: &[&downloads::Model], - mut bangumi: bangumi::Model, - ) -> eyre::Result { - if bangumi.save_path.is_none() { - let gen_sub_path = gen_bangumi_sub_path(&bangumi); - let mut bangumi_active = bangumi.into_active_model(); - bangumi_active.save_path = ActiveValue::Set(Some(gen_sub_path.to_string())); - bangumi = bangumi_active.update(db).await?; - } - - let sub_path = bangumi - .save_path - .as_ref() - .unwrap_or_else(|| unreachable!("must have a sub path")); - - let mut torrent_urls = vec![]; - for m in downloads.iter() { - torrent_urls.push(Url::parse(&m.url as &str)?); - } - - // make sequence to prevent too fast to be banned - for d in downloads.iter() { - let source = TorrentSource::parse(&d.url).await?; - self.add_torrents(source, sub_path.clone(), Some("bangumi")) - .await?; - } - - Ok(bangumi) - } -} - -pub async fn build_torrent_downloader_from_downloader_model( - model: downloaders::Model, -) -> eyre::Result> { - Ok(Box::new(match &model.category { - DownloaderCategory::QBittorrent => { - QBittorrentDownloader::from_downloader_model(model).await? - } - })) -} diff --git a/crates/recorder/src/lib.rs b/crates/recorder/src/lib.rs index 7702581..2711c7f 100644 --- a/crates/recorder/src/lib.rs +++ b/crates/recorder/src/lib.rs @@ -3,13 +3,13 @@ pub mod app; pub mod config; pub mod controllers; -pub mod dal; pub mod downloaders; pub mod migrations; pub mod models; pub mod parsers; pub mod path; pub mod search; +pub mod storage; pub mod tasks; pub mod views; pub mod workers; diff --git a/crates/recorder/src/migrations/defs.rs b/crates/recorder/src/migrations/defs.rs index c326ed6..23c1892 100644 --- a/crates/recorder/src/migrations/defs.rs +++ b/crates/recorder/src/migrations/defs.rs @@ -18,6 +18,7 @@ pub enum Subscribers { Pid, DisplayName, DownloaderId, + BangumiConf, } #[derive(DeriveIden)] @@ -36,18 +37,42 @@ pub enum Subscriptions { pub enum Bangumi { Table, Id, - DisplayName, SubscriptionId, + DisplayName, + OfficialTitle, + Fansub, + Season, + Filter, + PosterLink, + SavePath, + LastEp, + BangumiConfOverride, } #[derive(DeriveIden)] pub enum Episodes { Table, Id, + OriginTitle, + OfficialTitle, DisplayName, + NameZh, + NameJp, + NameEn, + SNameZh, + SNameJp, + SNameEn, BangumiId, - OutputName, DownloadId, + SavePath, + Resolution, + Season, + SeasonRaw, + Fansub, + PosterLink, + HomePage, + Subtitle, + Source, } #[derive(DeriveIden)] @@ -55,13 +80,14 @@ pub enum Downloads { Table, Id, SubscriptionId, - OriginalName, + OriginTitle, DisplayName, Status, CurrSize, AllSize, Mime, Url, + HomePage, } #[derive(DeriveIden)] @@ -73,7 +99,7 @@ pub enum Downloaders { Password, Username, SubscriberId, - DownloadPath, + SavePath, } #[async_trait::async_trait] diff --git a/crates/recorder/src/migrations/m20220101_000001_init.rs b/crates/recorder/src/migrations/m20220101_000001_init.rs index 21a6632..b7069f7 100644 --- a/crates/recorder/src/migrations/m20220101_000001_init.rs +++ b/crates/recorder/src/migrations/m20220101_000001_init.rs @@ -1,3 +1,4 @@ +use loco_rs::schema::jsonb_null; use sea_orm_migration::{prelude::*, schema::*}; use super::defs::{ @@ -20,6 +21,7 @@ impl MigrationTrait for Migration { .col(pk_auto(Subscribers::Id)) .col(string_len_uniq(Subscribers::Pid, 64)) .col(string(Subscribers::DisplayName)) + .col(jsonb_null(Subscribers::BangumiConf)) .to_owned(), ) .await?; @@ -84,8 +86,16 @@ impl MigrationTrait for Migration { .create_table( table_auto(Bangumi::Table) .col(pk_auto(Bangumi::Id)) - .col(text(Bangumi::DisplayName)) .col(integer(Bangumi::SubscriptionId)) + .col(text(Bangumi::DisplayName)) + .col(text(Bangumi::OfficialTitle)) + .col(string_null(Bangumi::Fansub)) + .col(unsigned(Bangumi::Season)) + .col(jsonb_null(Bangumi::Filter)) + .col(text_null(Bangumi::PosterLink)) + .col(text_null(Bangumi::SavePath)) + .col(unsigned(Bangumi::LastEp)) + .col(jsonb_null(Bangumi::BangumiConfOverride)) .foreign_key( ForeignKey::create() .name("fk_bangumi_subscription_id") @@ -94,6 +104,27 @@ impl MigrationTrait for Migration { .on_update(ForeignKeyAction::Restrict) .on_delete(ForeignKeyAction::Cascade), ) + .index( + Index::create() + .name("idx_bangumi_official_title") + .table(Bangumi::Table) + .col(Bangumi::OfficialTitle) + .unique(), + ) + .index( + Index::create() + .name("idx_bangumi_fansub") + .table(Bangumi::Table) + .col(Bangumi::Fansub) + .unique(), + ) + .index( + Index::create() + .name("idx_bangumi_display_name") + .table(Bangumi::Table) + .col(Bangumi::DisplayName) + .unique(), + ) .to_owned(), ) .await?; @@ -106,9 +137,26 @@ impl MigrationTrait for Migration { .create_table( table_auto(Episodes::Table) .col(pk_auto(Episodes::Id)) + .col(text(Episodes::OriginTitle)) + .col(text(Episodes::OfficialTitle)) .col(text(Episodes::DisplayName)) + .col(text_null(Episodes::NameZh)) + .col(text_null(Episodes::NameJp)) + .col(text_null(Episodes::NameEn)) + .col(text_null(Episodes::SNameZh)) + .col(text_null(Episodes::SNameJp)) + .col(text_null(Episodes::SNameEn)) .col(integer(Episodes::BangumiId)) - .col(text(Episodes::OutputName)) + .col(integer(Episodes::DownloadId)) + .col(text_null(Episodes::SavePath)) + .col(string_null(Episodes::Resolution)) + .col(integer(Episodes::Season)) + .col(string_null(Episodes::SeasonRaw)) + .col(string_null(Episodes::Fansub)) + .col(text_null(Episodes::PosterLink)) + .col(text_null(Episodes::HomePage)) + .col(jsonb_null(Episodes::Subtitle)) + .col(text_null(Episodes::Source)) .foreign_key( ForeignKey::create() .name("fk_episode_bangumi_id") @@ -117,6 +165,24 @@ impl MigrationTrait for Migration { .on_update(ForeignKeyAction::Restrict) .on_delete(ForeignKeyAction::Cascade), ) + .index( + Index::create() + .name("idx_episode_official_title") + .table(Episodes::Table) + .col(Episodes::OfficialTitle), + ) + .index( + Index::create() + .name("idx_episode_fansub") + .table(Episodes::Table) + .col(Episodes::Fansub), + ) + .index( + Index::create() + .name("idx_episode_display_name") + .table(Episodes::Table) + .col(Episodes::DisplayName), + ) .to_owned(), ) .await?; diff --git a/crates/recorder/src/migrations/m20240224_082543_add_downloads.rs b/crates/recorder/src/migrations/m20240224_082543_add_downloads.rs index 54fe73b..3c840c7 100644 --- a/crates/recorder/src/migrations/m20240224_082543_add_downloads.rs +++ b/crates/recorder/src/migrations/m20240224_082543_add_downloads.rs @@ -38,8 +38,8 @@ impl MigrationTrait for Migration { .create_table( table_auto(Downloads::Table) .col(pk_auto(Downloads::Id)) - .col(string(Downloads::OriginalName)) - .col(string(Downloads::DisplayName)) + .col(text(Downloads::OriginTitle)) + .col(text(Downloads::DisplayName)) .col(integer(Downloads::SubscriptionId)) .col(enumeration( Downloads::Status, @@ -51,15 +51,10 @@ impl MigrationTrait for Migration { DownloadMimeEnum, DownloadMime::iden_values(), )) - .col(big_unsigned(Downloads::AllSize)) - .col(big_unsigned(Downloads::CurrSize)) + .col(big_unsigned_null(Downloads::AllSize)) + .col(big_unsigned_null(Downloads::CurrSize)) .col(text(Downloads::Url)) - .index( - Index::create() - .table(Downloads::Table) - .col(Downloads::Url) - .name("idx_download_url"), - ) + .col(text_null(Downloads::HomePage)) .foreign_key( ForeignKey::create() .name("fk_download_subscription_id") @@ -68,6 +63,18 @@ impl MigrationTrait for Migration { .on_update(ForeignKeyAction::Restrict) .on_delete(ForeignKeyAction::Cascade), ) + .index( + Index::create() + .name("idx_download_url") + .table(Downloads::Table) + .col(Downloads::Url), + ) + .index( + Index::create() + .name("idx_download_home_page") + .table(Downloads::Table) + .col(Downloads::HomePage), + ) .to_owned(), ) .await?; diff --git a/crates/recorder/src/migrations/m20240225_060853_subscriber_add_downloader.rs b/crates/recorder/src/migrations/m20240225_060853_subscriber_add_downloader.rs index d969a61..e7cabb3 100644 --- a/crates/recorder/src/migrations/m20240225_060853_subscriber_add_downloader.rs +++ b/crates/recorder/src/migrations/m20240225_060853_subscriber_add_downloader.rs @@ -30,7 +30,7 @@ impl MigrationTrait for Migration { DownloaderCategoryEnum, DownloaderCategory::iden_values(), )) - .col(text(Downloaders::DownloadPath)) + .col(text(Downloaders::SavePath)) .col(integer(Downloaders::SubscriberId)) .foreign_key( ForeignKey::create() diff --git a/crates/recorder/src/models/bangumi.rs b/crates/recorder/src/models/bangumi.rs index b28d9a3..69e7730 100644 --- a/crates/recorder/src/models/bangumi.rs +++ b/crates/recorder/src/models/bangumi.rs @@ -1,8 +1,10 @@ +use std::collections::HashSet; + +use itertools::Itertools; use regex::Regex; use sea_orm::entity::prelude::*; pub use super::entities::bangumi::*; -use crate::models::downloads; #[async_trait::async_trait] impl ActiveModelBehavior for ActiveModel {} @@ -26,6 +28,29 @@ impl BangumiFilter { } impl Model { - pub async fn search_all() {} - pub async fn match_list(dnlds: Vec) {} + pub fn get_unique_key(&self) -> BangumiUniqueKey { + BangumiUniqueKey { + official_title: self.official_title.clone(), + season: self.season, + fansub: self.fansub.clone(), + } + } + + pub async fn find_by_unique_keys( + db: &DatabaseConnection, + unique_keys: impl Iterator, + ) -> eyre::Result> { + let unique_keys = unique_keys.collect::>(); + let mut found = Entity::find() + .filter(Column::OfficialTitle.is_in(unique_keys.iter().map(|k| &k.official_title))) + .all(db) + .await?; + + found = found + .into_iter() + .filter(|m| unique_keys.contains(&m.get_unique_key())) + .collect_vec(); + + Ok(found) + } } diff --git a/crates/recorder/src/models/db_utils.rs b/crates/recorder/src/models/db_utils.rs index 3e06a53..a8d2cd7 100644 --- a/crates/recorder/src/models/db_utils.rs +++ b/crates/recorder/src/models/db_utils.rs @@ -1,5 +1,5 @@ use sea_orm::{ - sea_query::{Expr, InsertStatement, IntoColumnRef, Query, SimpleExpr}, + sea_query::{Expr, InsertStatement, Query, SimpleExpr}, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, EntityName, EntityTrait, FromQueryResult, Iterable, SelectModel, SelectorRaw, TryGetable, }; diff --git a/crates/recorder/src/models/downloads.rs b/crates/recorder/src/models/downloads.rs index b2a22a8..d1a022d 100644 --- a/crates/recorder/src/models/downloads.rs +++ b/crates/recorder/src/models/downloads.rs @@ -1,65 +1,25 @@ -use itertools::Itertools; -use loco_rs::app::AppContext; -use sea_orm::{ - prelude::*, - sea_query::{InsertStatement, OnConflict}, -}; +use sea_orm::{prelude::*, ActiveValue}; pub use crate::models::entities::downloads::*; -use crate::{ - models::{ - db_utils::insert_many_with_returning_all, - subscriptions::{self, SubscriptionCategory}, - }, - parsers::mikan::{ - mikan_client::MikanClient, parse_mikan_rss_items_from_rss_link, MikanRssItem, - }, -}; +use crate::parsers::mikan::MikanRssItem; #[async_trait::async_trait] impl ActiveModelBehavior for ActiveModel {} impl ActiveModel { - pub fn from_mikan_rss_item(m: MikanRssItem, subscription_id: i32) -> Self { - todo!() - } -} - -impl Model { - pub async fn pull_subscription( - ctx: AppContext, - subscription: &subscriptions::Model, - ) -> eyre::Result> { - let db = &ctx.db; - match &subscription.category { - SubscriptionCategory::Mikan => { - let subscriber_id = subscription.subscriber_id; - let client = MikanClient::new(subscriber_id).await?; - let items = - parse_mikan_rss_items_from_rss_link(&client, &subscription.source_url).await?; - let all_items = items.collect::>(); - - if all_items.is_empty() { - return Ok(vec![]); - } - - let new_items = all_items - .into_iter() - .map(|i| ActiveModel::from_mikan_rss_item(i, subscription.id)) - .collect_vec(); - - // insert and filter out duplicated items - let new_items: Vec = - insert_many_with_returning_all(db, new_items, |stat: &mut InsertStatement| { - stat.on_conflict(OnConflict::column(Column::Url).do_nothing().to_owned()); - }) - .await?; - - Ok(new_items) - } - _ => { - todo!("other subscription categories") - } + pub fn from_mikan_rss_item(rss_item: MikanRssItem, subscription_id: i32) -> Self { + let download_mime = rss_item.get_download_mime(); + Self { + origin_title: ActiveValue::Set(rss_item.title.clone()), + display_name: ActiveValue::Set(rss_item.title), + subscription_id: ActiveValue::Set(subscription_id), + status: ActiveValue::Set(DownloadStatus::Pending), + mime: ActiveValue::Set(download_mime), + url: ActiveValue::Set(rss_item.url), + all_size: ActiveValue::Set(rss_item.content_length), + curr_size: ActiveValue::Set(Some(0)), + homepage: ActiveValue::Set(rss_item.homepage), + ..Default::default() } } } diff --git a/crates/recorder/src/models/entities/bangumi.rs b/crates/recorder/src/models/entities/bangumi.rs index 28e6198..32c10f8 100644 --- a/crates/recorder/src/models/entities/bangumi.rs +++ b/crates/recorder/src/models/entities/bangumi.rs @@ -37,7 +37,7 @@ pub enum BangumiRenameMethod { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)] -pub struct SubscriberBangumiConfigOverride { +pub struct SubscribeBangumiConfigOverride { pub leading_fansub_tag: Option, pub complete_history_episodes: Option, pub rename_method: Option, @@ -50,6 +50,13 @@ pub struct BangumiFilter { pub regex_filters: Option>, } +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub struct BangumiUniqueKey { + pub official_title: String, + pub season: u32, + pub fansub: Option, +} + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] #[sea_orm(table_name = "bangumi")] pub struct Model { @@ -60,15 +67,13 @@ pub struct Model { pub subscription_id: i32, pub display_name: String, pub official_title: String, - pub season: i32, - pub season_raw: Option, pub fansub: Option, + pub season: u32, pub filter: Option, - pub rss_link: Option, pub poster_link: Option, pub save_path: Option, - pub deleted: bool, - pub subscriber_conf_override: Option, + pub last_ep: u32, + pub bangumi_conf_override: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/crates/recorder/src/models/entities/downloaders.rs b/crates/recorder/src/models/entities/downloaders.rs index 89bba28..6b759c7 100644 --- a/crates/recorder/src/models/entities/downloaders.rs +++ b/crates/recorder/src/models/entities/downloaders.rs @@ -18,9 +18,7 @@ pub enum DownloaderCategory { #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] #[sea_orm(table_name = "downloaders")] pub struct Model { - #[sea_orm(column_type = "Timestamp")] pub created_at: DateTime, - #[sea_orm(column_type = "Timestamp")] pub updated_at: DateTime, #[sea_orm(primary_key)] pub id: i32, diff --git a/crates/recorder/src/models/entities/downloads.rs b/crates/recorder/src/models/entities/downloads.rs index c01a701..c939361 100644 --- a/crates/recorder/src/models/entities/downloads.rs +++ b/crates/recorder/src/models/entities/downloads.rs @@ -1,4 +1,4 @@ -use sea_orm::{entity::prelude::*, FromJsonQueryResult}; +use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; #[derive( @@ -41,7 +41,7 @@ pub struct Model { pub updated_at: DateTime, #[sea_orm(primary_key)] pub id: i32, - pub origin_name: String, + pub origin_title: String, pub display_name: String, pub subscription_id: i32, pub status: DownloadStatus, diff --git a/crates/recorder/src/models/entities/episodes.rs b/crates/recorder/src/models/entities/episodes.rs index 2c0ae43..0f0b1fa 100644 --- a/crates/recorder/src/models/entities/episodes.rs +++ b/crates/recorder/src/models/entities/episodes.rs @@ -10,7 +10,7 @@ pub struct Model { pub updated_at: DateTime, #[sea_orm(primary_key)] pub id: i32, - pub raw_name: String, + pub origin_title: String, pub official_title: String, pub display_name: String, pub name_zh: Option, @@ -20,16 +20,15 @@ pub struct Model { pub s_name_jp: Option, pub s_name_en: Option, pub bangumi_id: i32, - pub download_id: i32, - pub save_path: String, + pub download_id: Option, + pub save_path: Option, pub resolution: Option, - pub season: i32, + pub season: u32, pub season_raw: Option, pub fansub: Option, pub poster_link: Option, pub home_page: Option, pub subtitle: Option>, - pub deleted: bool, pub source: Option, } diff --git a/crates/recorder/src/models/entities/subscribers.rs b/crates/recorder/src/models/entities/subscribers.rs index 0fba462..6ac3c2a 100644 --- a/crates/recorder/src/models/entities/subscribers.rs +++ b/crates/recorder/src/models/entities/subscribers.rs @@ -6,14 +6,14 @@ use serde::{Deserialize, Serialize}; use super::bangumi::BangumiRenameMethod; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)] -pub struct SubscriberBangumiConfig { +pub struct SubscribeBangumiConfig { pub leading_fansub_tag: bool, pub complete_history_episodes: bool, pub rename_method: BangumiRenameMethod, pub remove_bad_torrent: bool, } -impl Default for SubscriberBangumiConfig { +impl Default for SubscribeBangumiConfig { fn default() -> Self { Self { leading_fansub_tag: false, @@ -35,7 +35,7 @@ pub struct Model { pub pid: String, pub display_name: String, pub downloader_id: Option, - pub bangumi_conf: SubscriberBangumiConfig, + pub bangumi_conf: SubscribeBangumiConfig, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/crates/recorder/src/models/episodes.rs b/crates/recorder/src/models/episodes.rs index 7d9e9dc..c05e6f8 100644 --- a/crates/recorder/src/models/episodes.rs +++ b/crates/recorder/src/models/episodes.rs @@ -1,37 +1,43 @@ use sea_orm::{entity::prelude::*, ActiveValue}; pub use super::entities::episodes::*; -use crate::models::{bangumi, downloads}; +use crate::{ + models::downloads, + parsers::{mikan::MikanEpisodeMeta, raw::RawEpisodeMeta}, +}; #[async_trait::async_trait] impl ActiveModelBehavior for ActiveModel {} impl ActiveModel { - pub async fn from_mikan_rss_item(dl: &downloads::Model, bgm: &bangumi::Model) -> Self { - let _ = Self { - raw_name: ActiveValue::Set(dl.origin_name.clone()), - official_title: ActiveValue::Set(bgm.official_title.clone()), - display_name: ActiveValue::Set(bgm.display_name.clone()), - name_zh: Default::default(), - name_jp: Default::default(), - name_en: Default::default(), - s_name_zh: Default::default(), - s_name_jp: Default::default(), - s_name_en: Default::default(), - bangumi_id: Default::default(), - download_id: Default::default(), - save_path: Default::default(), - resolution: Default::default(), - season: Default::default(), - season_raw: Default::default(), - fansub: Default::default(), - poster_link: Default::default(), - home_page: Default::default(), - subtitle: Default::default(), - deleted: Default::default(), - source: Default::default(), + pub fn from_mikan_meta( + bangumi_id: i32, + dl: downloads::Model, + raw_meta: RawEpisodeMeta, + mikan_meta: MikanEpisodeMeta, + mikan_poster: Option, + ) -> Self { + Self { + origin_title: ActiveValue::Set(dl.origin_title), + official_title: ActiveValue::Set(mikan_meta.official_title.clone()), + display_name: ActiveValue::Set(mikan_meta.official_title), + name_zh: ActiveValue::Set(raw_meta.name_zh), + name_jp: ActiveValue::Set(raw_meta.name_jp), + name_en: ActiveValue::Set(raw_meta.name_en), + s_name_zh: ActiveValue::Set(raw_meta.s_name_zh), + s_name_jp: ActiveValue::Set(raw_meta.s_name_jp), + s_name_en: ActiveValue::Set(raw_meta.s_name_en), + bangumi_id: ActiveValue::Set(bangumi_id), + download_id: ActiveValue::Set(Some(dl.id)), + resolution: ActiveValue::Set(raw_meta.resolution), + season: ActiveValue::Set(raw_meta.season), + season_raw: ActiveValue::Set(raw_meta.season_raw), + fansub: ActiveValue::Set(raw_meta.fansub), + poster_link: ActiveValue::Set(mikan_poster), + home_page: ActiveValue::Set(dl.homepage), + subtitle: ActiveValue::Set(raw_meta.sub), + source: ActiveValue::Set(raw_meta.source), ..Default::default() - }; - todo!() + } } } diff --git a/crates/recorder/src/models/subscriptions.rs b/crates/recorder/src/models/subscriptions.rs index 2da4abb..17eb55d 100644 --- a/crates/recorder/src/models/subscriptions.rs +++ b/crates/recorder/src/models/subscriptions.rs @@ -1,7 +1,28 @@ -use sea_orm::{entity::prelude::*, ActiveValue}; +use std::collections::HashMap; + +use itertools::Itertools; +use loco_rs::app::AppContext; +use sea_orm::{ + entity::prelude::*, + sea_query::{InsertStatement, OnConflict}, + ActiveValue, +}; use serde::{Deserialize, Serialize}; +use tracing::{event, instrument, Level}; pub use super::entities::subscriptions::{self, *}; +use crate::{ + models::{bangumi, db_utils::insert_many_with_returning_all, downloads, episodes}, + parsers::{ + mikan::{ + parse_episode_meta_from_mikan_homepage, parse_mikan_rss_items_from_rss_link, + MikanClient, MikanEpisodeMeta, + }, + raw::{parse_episode_meta_from_raw_name, RawEpisodeMeta}, + }, + path::extract_extname_from_url, + storage::{AppContextDalExt, DalContentType}, +}; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SubscriptionCreateFromRssDto { @@ -80,4 +101,190 @@ impl Model { .await?; Ok(()) } + + #[instrument( + fields(subscriber_id = "self.subscriber_id", subscription_id = "self.id"), + skip(self, db, ctx) + )] + pub async fn pull_item(&self, db: &DatabaseConnection, ctx: &AppContext) -> eyre::Result<()> { + let subscription = self; + let subscription_id = subscription.id; + match &subscription.category { + SubscriptionCategory::Mikan => { + let subscriber_id = subscription.subscriber_id; + let mikan_client = MikanClient::new(subscriber_id).await?; + let mikan_rss_items = + parse_mikan_rss_items_from_rss_link(&mikan_client, &subscription.source_url) + .await?; + let all_items = mikan_rss_items.collect::>(); + + if all_items.is_empty() { + return Ok(()); + } + + let new_downloads = all_items + .into_iter() + .map(|rss_item| { + downloads::ActiveModel::from_mikan_rss_item(rss_item, subscription.id) + }) + .collect_vec(); + + // insert and filter out duplicated items + let new_downloads: Vec = insert_many_with_returning_all( + db, + new_downloads, + |stat: &mut InsertStatement| { + stat.on_conflict( + OnConflict::column(downloads::Column::Url) + .do_nothing() + .to_owned(), + ); + }, + ) + .await?; + + pub struct MikanEpMetaBundle { + pub download: downloads::Model, + pub mikan: MikanEpisodeMeta, + pub raw: RawEpisodeMeta, + pub poster: Option, + } + + let mut ep_metas: HashMap> = + HashMap::new(); + let dal = ctx.get_dal_unwrap().await; + { + for dl in new_downloads { + let mut mikan_meta = if let Some(homepage) = dl.homepage.as_deref() { + match parse_episode_meta_from_mikan_homepage(&mikan_client, homepage) + .await + { + Ok(mikan_meta) => mikan_meta, + Err(e) => { + let error: &dyn std::error::Error = e.as_ref(); + event!( + Level::ERROR, + desc = "failed to parse episode meta from mikan homepage", + homepage = homepage, + error = error + ); + continue; + } + } + } else { + continue; + }; + let mikan_poster_link = if let Some(poster) = mikan_meta.poster.take() { + if let Some(extname) = extract_extname_from_url(&poster.origin_url) { + let result = dal + .store_blob( + DalContentType::Poster, + &extname, + poster.data, + &subscriber_id.to_string(), + ) + .await; + match result { + Ok(stored_url) => Some(stored_url.to_string()), + Err(e) => { + let error: &dyn std::error::Error = e.as_ref(); + event!( + Level::ERROR, + desc = "failed to store mikan meta poster", + origin_url = poster.origin_url.as_str(), + error = error + ); + None + } + } + } else { + event!( + Level::ERROR, + desc = "failed to extract mikan meta poster extname", + origin_url = poster.origin_url.as_str(), + ); + None + } + } else { + None + }; + let raw_meta = match parse_episode_meta_from_raw_name(&dl.origin_title) { + Ok(raw_meta) => raw_meta, + Err(e) => { + let error: &dyn std::error::Error = e.as_ref(); + event!( + Level::ERROR, + desc = "failed to parse episode meta from origin name", + origin_name = &dl.origin_title, + error = error + ); + continue; + } + }; + let key = bangumi::BangumiUniqueKey { + official_title: mikan_meta.official_title.clone(), + season: raw_meta.season, + fansub: raw_meta.fansub.clone(), + }; + let meta = MikanEpMetaBundle { + download: dl, + mikan: mikan_meta, + raw: raw_meta, + poster: mikan_poster_link, + }; + ep_metas.entry(key).or_default().push(meta); + } + } + + for (_, eps) in ep_metas { + let meta = eps.first().unwrap_or_else(|| { + unreachable!( + "subscriptions pull items bangumi must have at least one episode meta" + ) + }); + let last_ep = eps.iter().fold(0, |acc, ep| acc.max(ep.raw.episode_index)); + let official_title = &meta.mikan.official_title; + let bgm = bangumi::ActiveModel { + subscription_id: ActiveValue::Set(subscription_id), + display_name: ActiveValue::Set(official_title.clone()), + official_title: ActiveValue::Set(official_title.clone()), + fansub: ActiveValue::Set(meta.raw.fansub.clone()), + season: ActiveValue::Set(meta.raw.season), + poster_link: ActiveValue::Set(meta.poster.clone()), + last_ep: ActiveValue::Set(last_ep), + ..Default::default() + }; + + let bgm = bangumi::Entity::insert(bgm) + .on_conflict( + OnConflict::columns([ + bangumi::Column::OfficialTitle, + bangumi::Column::Season, + bangumi::Column::Fansub, + ]) + .update_columns([bangumi::Column::LastEp]) + .to_owned(), + ) + .exec_with_returning(db) + .await?; + + let eps = eps.into_iter().map(|ep| { + episodes::ActiveModel::from_mikan_meta( + bgm.id, + ep.download, + ep.raw, + ep.mikan, + ep.poster, + ) + }); + episodes::Entity::insert_many(eps).exec(db).await?; + } + + Ok(()) + } + _ => { + todo!("other subscription categories") + } + } + } } diff --git a/crates/recorder/src/parsers/errors.rs b/crates/recorder/src/parsers/errors.rs index 0ecb06a..0d1a833 100644 --- a/crates/recorder/src/parsers/errors.rs +++ b/crates/recorder/src/parsers/errors.rs @@ -16,4 +16,6 @@ pub enum ParseError { LanguageTagError(#[from] oxilangtag::LanguageTagParseError), #[error("Unsupported language preset: {0}")] UnsupportedLanguagePreset(String), + #[error("Parse episode meta error, get empty official title, homepage = {0}")] + MikanEpisodeMetaEmptyOfficialTitleError(String), } diff --git a/crates/recorder/src/parsers/mikan/mikan_client.rs b/crates/recorder/src/parsers/mikan/mikan_client.rs index e705feb..7215a6c 100644 --- a/crates/recorder/src/parsers/mikan/mikan_client.rs +++ b/crates/recorder/src/parsers/mikan/mikan_client.rs @@ -2,7 +2,7 @@ use std::{ops::Deref, sync::Arc}; use tokio::sync::OnceCell; -use crate::downloaders::defs::ApiClient; +use crate::downloaders::ApiClient; pub struct MikanClient { api_client: ApiClient, diff --git a/crates/recorder/src/parsers/mikan/mikan_ep_parser.rs b/crates/recorder/src/parsers/mikan/mikan_ep_parser.rs index 855debd..53e75dc 100644 --- a/crates/recorder/src/parsers/mikan/mikan_ep_parser.rs +++ b/crates/recorder/src/parsers/mikan/mikan_ep_parser.rs @@ -3,17 +3,26 @@ use html_escape::decode_html_entities; use lazy_static::lazy_static; use lightningcss::{properties::Property, values::image::Image}; use regex::Regex; +use reqwest::IntoUrl; +use tracing::instrument; use url::Url; use crate::parsers::{ + errors::ParseError, html::{get_tag_style, query_selector_first_tag}, mikan::mikan_client::MikanClient, }; +#[derive(Clone, Debug)] +pub struct MikanEpisodeMetaPosterBlob { + pub origin_url: Url, + pub data: Bytes, +} + +#[derive(Clone, Debug)] pub struct MikanEpisodeMeta { pub homepage: Url, - pub poster_data: Option, - pub origin_poster_src: Option, + pub poster: Option, pub official_title: String, } @@ -21,12 +30,14 @@ lazy_static! { static ref MIKAN_TITLE_SEASON: Regex = Regex::new("第.*季").unwrap(); } +#[instrument(skip(client, url))] pub async fn parse_episode_meta_from_mikan_homepage( client: &MikanClient, - url: Url, -) -> eyre::Result> { + url: impl IntoUrl, +) -> eyre::Result { + let url = url.into_url()?; let url_host = url.origin().unicode_serialization(); - let content = client.fetch_text(|f| f.get(url.as_str())).await?; + let content = client.fetch_text(|f| f.get(url.clone())).await?; let dom = tl::parse(&content, tl::ParserOptions::default())?; let parser = dom.parser(); let poster_node = query_selector_first_tag(&dom, r"div.bangumi-poster", parser); @@ -62,12 +73,19 @@ pub async fn parse_episode_meta_from_mikan_homepage( p.set_query(None); p }); - let poster_data = if let Some(p) = origin_poster_src.as_ref() { - client.fetch_bytes(|f| f.get(p.clone())).await.ok() + let poster = if let Some(p) = origin_poster_src { + client + .fetch_bytes(|f| f.get(p.clone())) + .await + .ok() + .map(|data| MikanEpisodeMetaPosterBlob { + data, + origin_url: p, + }) } else { None }; - let meta = official_title_node + let official_title = official_title_node .map(|s| s.inner_text(parser)) .and_then(|official_title| { let title = MIKAN_TITLE_SEASON @@ -80,13 +98,13 @@ pub async fn parse_episode_meta_from_mikan_homepage( Some(title) } }) - .map(|title| MikanEpisodeMeta { - homepage: url, - poster_data, - official_title: title, - origin_poster_src, - }); - Ok(meta) + .ok_or_else(|| ParseError::MikanEpisodeMetaEmptyOfficialTitleError(url.to_string()))?; + + Ok(MikanEpisodeMeta { + homepage: url, + poster, + official_title, + }) } #[cfg(test)] @@ -105,24 +123,25 @@ mod test { let client = MikanClient::new(0).await.expect("should get mikan client"); - if let Some(ep_meta) = - parse_episode_meta_from_mikan_homepage(&client, url.clone()).await? + let ep_meta = parse_episode_meta_from_mikan_homepage(&client, url.clone()).await?; { assert_eq!(ep_meta.homepage, url); assert_eq!(ep_meta.official_title, "葬送的芙莉莲"); assert_eq!( - ep_meta.origin_poster_src, + ep_meta.poster.clone().map(|p| p.origin_url), Some(Url::parse( "https://mikanani.me/images/Bangumi/202309/5ce9fed1.jpg" )?) ); - let u8_data = ep_meta.poster_data.expect("should have poster data"); + let u8_data = ep_meta + .poster + .clone() + .map(|p| p.data) + .expect("should have poster data"); assert!( u8_data.starts_with(&[255, 216, 255, 224]), "should start with valid jpeg data magic number" ); - } else { - panic!("can not find mikan episode title") } Ok(()) diff --git a/crates/recorder/src/parsers/mikan/mikan_rss_parser.rs b/crates/recorder/src/parsers/mikan/mikan_rss_parser.rs index a062529..134f8d3 100644 --- a/crates/recorder/src/parsers/mikan/mikan_rss_parser.rs +++ b/crates/recorder/src/parsers/mikan/mikan_rss_parser.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::{ downloaders::defs::BITTORRENT_MIME_TYPE, + models::prelude::DownloadMime, parsers::{errors::ParseError, mikan::mikan_client::MikanClient}, }; @@ -17,6 +18,12 @@ pub struct MikanRssItem { pub pub_date: Option, } +impl MikanRssItem { + pub fn get_download_mime(&self) -> DownloadMime { + DownloadMime::BitTorrent + } +} + impl TryFrom for MikanRssItem { type Error = ParseError; diff --git a/crates/recorder/src/parsers/mikan/mod.rs b/crates/recorder/src/parsers/mikan/mod.rs index bb6736d..0dc8254 100644 --- a/crates/recorder/src/parsers/mikan/mod.rs +++ b/crates/recorder/src/parsers/mikan/mod.rs @@ -2,5 +2,6 @@ pub mod mikan_client; pub mod mikan_ep_parser; pub mod mikan_rss_parser; +pub use mikan_client::MikanClient; pub use mikan_ep_parser::{parse_episode_meta_from_mikan_homepage, MikanEpisodeMeta}; pub use mikan_rss_parser::{parse_mikan_rss_items_from_rss_link, MikanRssItem}; diff --git a/crates/recorder/src/parsers/raw/raw_ep_parser.rs b/crates/recorder/src/parsers/raw/raw_ep_parser.rs index 8dcc02e..0fdefb3 100644 --- a/crates/recorder/src/parsers/raw/raw_ep_parser.rs +++ b/crates/recorder/src/parsers/raw/raw_ep_parser.rs @@ -43,19 +43,19 @@ lazy_static! { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct RawEpisodeMeta { - name_en: Option, - name_en_no_season: Option, - name_jp: Option, - name_jp_no_season: Option, - name_zh: Option, - name_zh_no_season: Option, - season: i32, - season_raw: Option, - episode_index: i32, - sub: Option, - source: Option, - fansub: Option, - resolution: Option, + pub name_en: Option, + pub s_name_en: Option, + pub name_jp: Option, + pub s_name_jp: Option, + pub name_zh: Option, + pub s_name_zh: Option, + pub season: u32, + pub season_raw: Option, + pub episode_index: u32, + pub sub: Option>, + pub source: Option, + pub fansub: Option, + pub resolution: Option, } fn extract_fansub(raw_name: &str) -> Option<&str> { @@ -110,7 +110,7 @@ fn title_body_pre_process(title_body: &str, fansub: Option<&str>) -> eyre::Resul Ok(raw.to_string()) } -fn extract_season_from_title_body(title_body: &str) -> (String, Option, i32) { +fn extract_season_from_title_body(title_body: &str) -> (String, Option, u32) { let name_and_season = EN_BRACKET_SPLIT_RE.replace_all(title_body, " "); let seasons = SEASON_EXTRACT_SEASON_ALL_RE .find(&name_and_season) @@ -122,7 +122,7 @@ fn extract_season_from_title_body(title_body: &str) -> (String, Option, return (title_body.to_string(), None, 1); } - let mut season = 1; + let mut season = 1u32; let mut season_raw = None; let name = SEASON_EXTRACT_SEASON_ALL_RE.replace_all(&name_and_season, ""); @@ -131,7 +131,7 @@ fn extract_season_from_title_body(title_body: &str) -> (String, Option, if let Some(m) = SEASON_EXTRACT_SEASON_EN_PREFIX_RE.find(s) { if let Ok(s) = SEASON_EXTRACT_SEASON_ALL_RE .replace_all(m.as_str(), "") - .parse::() + .parse::() { season = s; break; @@ -140,7 +140,7 @@ fn extract_season_from_title_body(title_body: &str) -> (String, Option, if let Some(m) = SEASON_EXTRACT_SEASON_EN_NTH_RE.find(s) { if let Some(s) = DIGIT_1PLUS_REG .find(m.as_str()) - .and_then(|s| s.as_str().parse::().ok()) + .and_then(|s| s.as_str().parse::().ok()) { season = s; break; @@ -149,13 +149,13 @@ fn extract_season_from_title_body(title_body: &str) -> (String, Option, if let Some(m) = SEASON_EXTRACT_SEASON_ZH_PREFIX_RE.find(s) { if let Ok(s) = SEASON_EXTRACT_SEASON_ZH_PREFIX_SUB_RE .replace(m.as_str(), "") - .parse::() + .parse::() { season = s; break; } if let Some(m) = ZH_NUM_RE.find(m.as_str()) { - season = ZH_NUM_MAP[m.as_str()]; + season = ZH_NUM_MAP[m.as_str()] as u32; break; } } @@ -207,21 +207,25 @@ fn extract_name_from_title_body_name_section( (name_en, name_zh, name_jp) } -fn extract_episode_index_from_title_episode(title_episode: &str) -> Option { +fn extract_episode_index_from_title_episode(title_episode: &str) -> Option { DIGIT_1PLUS_REG .find(title_episode)? .as_str() - .parse::() + .parse::() .ok() } -fn clear_sub(sub: Option) -> Option { - sub.map(|s| CLEAR_SUB_RE.replace_all(&s, "").to_string()) +fn clear_sub(sub: Option>) -> Option> { + sub.map(|s| { + s.into_iter() + .map(|s| CLEAR_SUB_RE.replace_all(&s, "").to_string()) + .collect_vec() + }) } fn extract_tags_from_title_extra( title_extra: &str, -) -> (Option, Option, Option) { +) -> (Option>, Option, Option) { let replaced = TAGS_EXTRACT_SPLIT_RE.replace_all(title_extra, " "); let elements = replaced .split(' ') @@ -229,12 +233,19 @@ fn extract_tags_from_title_extra( .filter(|s| !s.is_empty()) .collect_vec(); - let mut sub = None; + let mut sub: Option> = None; let mut resolution = None; let mut source = None; for element in elements.iter() { if SUB_RE.is_match(element) { - sub = Some(element.to_string()) + let el = element.to_string(); + sub = Some(match sub { + Some(mut res) => { + res.push(el); + res + } + None => vec![el], + }) } else if RESOLUTION_RE.is_match(element) { resolution = Some(element.to_string()) } else if SOURCE_L1_RE.is_match(element) { @@ -292,11 +303,11 @@ pub fn parse_episode_meta_from_raw_name(s: &str) -> eyre::Result let (sub, resolution, source) = extract_tags_from_title_extra(title_extra); Ok(RawEpisodeMeta { name_en, - name_en_no_season, + s_name_en: name_en_no_season, name_jp, - name_jp_no_season, + s_name_jp: name_jp_no_season, name_zh, - name_zh_no_season, + s_name_zh: name_zh_no_season, season, season_raw, episode_index, diff --git a/crates/recorder/src/parsers/tmdb/tmdb_client.rs b/crates/recorder/src/parsers/tmdb/tmdb_client.rs index 9bc1ec1..76fa6b7 100644 --- a/crates/recorder/src/parsers/tmdb/tmdb_client.rs +++ b/crates/recorder/src/parsers/tmdb/tmdb_client.rs @@ -8,7 +8,7 @@ use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION}; use tokio::sync::RwLock; use weak_table::WeakValueHashMap; -use crate::downloaders::defs::{ApiClient, DEFAULT_USER_AGENT}; +use crate::downloaders::ApiClient; pub(crate) const TMDB_API_ORIGIN: &str = "https://api.themoviedb.org"; diff --git a/crates/recorder/src/path/mod.rs b/crates/recorder/src/path/mod.rs index f8de902..2a881d2 100644 --- a/crates/recorder/src/path/mod.rs +++ b/crates/recorder/src/path/mod.rs @@ -1 +1,4 @@ pub mod torrent_path; +pub mod url_utils; + +pub use url_utils::{extract_extname_from_url, extract_filename_from_url}; diff --git a/crates/recorder/src/path/torrent_path.rs b/crates/recorder/src/path/torrent_path.rs index fc24526..1108407 100644 --- a/crates/recorder/src/path/torrent_path.rs +++ b/crates/recorder/src/path/torrent_path.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use quirks_path::{Path, PathBuf}; use crate::{ - downloaders::defs::Torrent, + downloaders::torrent::Torrent, models::{bangumi, subscribers}, parsers::defs::SEASON_REGEX, }; @@ -70,7 +70,7 @@ pub fn gen_bangumi_sub_path(data: &bangumi::Model) -> PathBuf { PathBuf::from(data.official_title.to_string()).join(format!("Season {}", data.season)) } -pub fn rule_name(bgm: &bangumi::Model, conf: &subscribers::SubscriberBangumiConfig) -> String { +pub fn rule_name(bgm: &bangumi::Model, conf: &subscribers::SubscribeBangumiConfig) -> String { if let (true, Some(group_name)) = (conf.leading_fansub_tag, &bgm.fansub) { format!("[{}] {} S{}", group_name, bgm.official_title, bgm.season) } else { diff --git a/crates/recorder/src/path/url_utils.rs b/crates/recorder/src/path/url_utils.rs new file mode 100644 index 0000000..1d2112e --- /dev/null +++ b/crates/recorder/src/path/url_utils.rs @@ -0,0 +1,19 @@ +use quirks_path::Path; +use url::Url; + +pub fn extract_filename_from_url(url: &Url) -> Option<&str> { + url.path_segments().and_then(|s| s.last()).and_then(|last| { + if last.is_empty() { + Some(last) + } else { + None + } + }) +} + +pub fn extract_extname_from_url(url: &Url) -> Option { + let filename = extract_filename_from_url(url); + filename + .and_then(|f| Path::new(f).extension()) + .map(|ext| format!(".{}", ext)) +} diff --git a/crates/recorder/src/dal/mod.rs b/crates/recorder/src/storage/dal.rs similarity index 67% rename from crates/recorder/src/dal/mod.rs rename to crates/recorder/src/storage/dal.rs index 7e45f96..89c5ac3 100644 --- a/crates/recorder/src/dal/mod.rs +++ b/crates/recorder/src/storage/dal.rs @@ -1,3 +1,5 @@ +use std::fmt::Display; + use bytes::Bytes; use opendal::{layers::LoggingLayer, services, Operator}; use quirks_path::{Path, PathBuf}; @@ -9,11 +11,11 @@ use crate::config::AppDalConf; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub enum AppDalContentCategory { +pub enum DalContentType { Poster, } -impl AsRef for AppDalContentCategory { +impl AsRef for DalContentType { fn as_ref(&self) -> &str { match self { Self::Poster => "poster", @@ -22,7 +24,7 @@ impl AsRef for AppDalContentCategory { } #[derive(Debug, Clone)] -pub struct AppDalContext { +pub struct DalContext { pub config: AppDalConf, } @@ -31,16 +33,35 @@ pub enum DalStoredUrl { Absolute { url: Url }, } -impl AppDalContext { - pub fn new(app_dal_conf: AppDalConf) -> Self { - Self { - config: app_dal_conf, +impl DalStoredUrl { + pub fn as_str(&self) -> &str { + match self { + Self::RelativePath { path } => path.as_str(), + Self::Absolute { url } => url.as_str(), } } +} + +impl AsRef for DalStoredUrl { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl Display for DalStoredUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str().to_string()) + } +} + +impl DalContext { + pub fn new(dal_conf: AppDalConf) -> Self { + Self { config: dal_conf } + } pub async fn store_blob( &self, - content_category: AppDalContentCategory, + content_category: DalContentType, extname: &str, data: Bytes, subscriber_pid: &str, diff --git a/crates/recorder/src/storage/dal_ext.rs b/crates/recorder/src/storage/dal_ext.rs new file mode 100644 index 0000000..bb80cae --- /dev/null +++ b/crates/recorder/src/storage/dal_ext.rs @@ -0,0 +1,40 @@ +use std::sync::Arc; + +use eyre::Context; +use loco_rs::app::AppContext; +use tokio::sync::OnceCell; + +use crate::{ + config::{deserialize_key_path_from_loco_rs_config, AppDalConf}, + storage::DalContext, +}; + +static APP_DAL_CONTEXT: OnceCell> = OnceCell::const_new(); + +#[async_trait::async_trait] +pub trait AppContextDalExt { + async fn get_dal(&self) -> eyre::Result>; + async fn get_dal_unwrap(&self) -> Arc; + async fn init_dal(&self) -> eyre::Result> { + self.get_dal().await.wrap_err("dal context failed to init") + } +} + +#[async_trait::async_trait] +impl AppContextDalExt for AppContext { + async fn get_dal(&self) -> eyre::Result> { + let context = APP_DAL_CONTEXT + .get_or_try_init(|| async { + deserialize_key_path_from_loco_rs_config::(&["dal"], &self.config) + .map(|dal_conf| Arc::new(DalContext::new(dal_conf))) + }) + .await?; + Ok(context.clone()) + } + + async fn get_dal_unwrap(&self) -> Arc { + self.get_dal() + .await + .unwrap_or_else(|e| panic!("dal context failed to init: {}", e)) + } +} diff --git a/crates/recorder/src/storage/dal_initializer.rs b/crates/recorder/src/storage/dal_initializer.rs new file mode 100644 index 0000000..0b22e3f --- /dev/null +++ b/crates/recorder/src/storage/dal_initializer.rs @@ -0,0 +1,26 @@ +use axum::Router as AxumRouter; +use loco_rs::app::{AppContext, Initializer}; + +use crate::storage::AppContextDalExt; + +pub struct AppDalInitializer; + +#[async_trait::async_trait] +impl Initializer for AppDalInitializer { + fn name(&self) -> String { + "AppDalInitializer".to_string() + } + + async fn before_run(&self, ctx: &AppContext) -> loco_rs::Result<()> { + ctx.init_dal().await?; + Ok(()) + } + + async fn after_routes( + &self, + router: AxumRouter, + _ctx: &AppContext, + ) -> loco_rs::Result { + Ok(router) + } +} diff --git a/crates/recorder/src/storage/mod.rs b/crates/recorder/src/storage/mod.rs new file mode 100644 index 0000000..d4bb8e8 --- /dev/null +++ b/crates/recorder/src/storage/mod.rs @@ -0,0 +1,7 @@ +pub mod dal; +pub mod dal_ext; +pub mod dal_initializer; + +pub use dal::{DalContentType, DalContext, DalStoredUrl}; +pub use dal_ext::AppContextDalExt; +pub use dal_initializer::AppDalInitializer; diff --git a/crates/recorder/src/workers/collect.rs b/crates/recorder/src/workers/collect.rs index 197f9cb..c38f6e9 100644 --- a/crates/recorder/src/workers/collect.rs +++ b/crates/recorder/src/workers/collect.rs @@ -2,7 +2,7 @@ use loco_rs::prelude::*; use serde::{Deserialize, Serialize}; use tracing::info; -use crate::models::{bangumi, subscribers}; +use crate::models::bangumi; pub struct CollectHistoryEpisodesWorker { pub ctx: AppContext, @@ -14,11 +14,12 @@ pub enum CollectHistoryEpisodesWorkerArgs { } impl CollectHistoryEpisodesWorker { - pub async fn collect_history_episodes(bangumi: &bangumi::Model, only_season: bool) { + pub async fn collect_history_episodes(bangumi: &bangumi::Model, _only_season: bool) { info!( "Start collecting {} season {}...", bangumi.official_title, bangumi.season ); + todo!() } } @@ -30,7 +31,7 @@ impl worker::AppWorker for CollectHistoryEpiso #[async_trait] impl worker::Worker for CollectHistoryEpisodesWorker { - async fn perform(&self, args: CollectHistoryEpisodesWorkerArgs) -> worker::Result<()> { + async fn perform(&self, _args: CollectHistoryEpisodesWorkerArgs) -> worker::Result<()> { println!("================================================"); let db = &self.ctx.db; diff --git a/crates/recorder/src/workers/subscription.rs b/crates/recorder/src/workers/subscription.rs index 7c4b5a9..dd382ac 100644 --- a/crates/recorder/src/workers/subscription.rs +++ b/crates/recorder/src/workers/subscription.rs @@ -20,11 +20,11 @@ impl worker::AppWorker for SubscriptionWorker { #[async_trait] impl worker::Worker for SubscriptionWorker { - async fn perform(&self, args: SubscriptionWorkerArgs) -> worker::Result<()> { + async fn perform(&self, _args: SubscriptionWorkerArgs) -> worker::Result<()> { println!("================================================"); - let db = &self.ctx.db; - let storage = &self.ctx.storage; + // let db = &self.ctx.db; + // let storage = &self.ctx.storage; println!("================================================"); Ok(())