From f245a687909dc1f6be75a6395ba699ddf482768c Mon Sep 17 00:00:00 2001 From: lonelyhentxi Date: Sun, 4 May 2025 03:59:59 +0800 Subject: [PATCH] fix: fix some issues --- apps/recorder/src/extract/mikan/client.rs | 43 +++++++++++-------- apps/recorder/src/extract/mikan/credential.rs | 20 +++++++++ apps/recorder/src/extract/mikan/mod.rs | 4 +- apps/recorder/src/extract/mikan/web.rs | 32 +++++++++++--- apps/recorder/src/models/credential_3rd.rs | 2 +- apps/recorder/src/test_utils/crypto.rs | 9 ++++ apps/recorder/src/test_utils/database.rs | 3 +- apps/recorder/src/test_utils/mod.rs | 1 + apps/recorder/src/test_utils/tracing.rs | 3 +- justfile | 8 +++- packages/fetch/src/client/core.rs | 6 +-- packages/fetch/src/core.rs | 2 +- packages/fetch/src/lib.rs | 2 +- 13 files changed, 99 insertions(+), 36 deletions(-) create mode 100644 apps/recorder/src/extract/mikan/credential.rs create mode 100644 apps/recorder/src/test_utils/crypto.rs diff --git a/apps/recorder/src/extract/mikan/client.rs b/apps/recorder/src/extract/mikan/client.rs index 11bfd6b..112bf72 100644 --- a/apps/recorder/src/extract/mikan/client.rs +++ b/apps/recorder/src/extract/mikan/client.rs @@ -2,12 +2,11 @@ use std::{fmt::Debug, ops::Deref, sync::Arc}; use fetch::{HttpClient, HttpClientTrait}; use maplit::hashmap; -use sea_orm::DbErr; -use serde::{Deserialize, Serialize}; +use sea_orm::{ActiveModelTrait, ActiveValue::Set, DbErr, TryIntoModel}; use url::Url; use util::OptDynErr; -use super::{MikanConfig, constants::MIKAN_ACCOUNT_MANAGE_PAGE_PATH}; +use super::{MikanConfig, MikanCredentialForm, constants::MIKAN_ACCOUNT_MANAGE_PAGE_PATH}; use crate::{ app::AppContextTrait, crypto::UserPassCredential, @@ -15,22 +14,6 @@ use crate::{ extract::mikan::constants::{MIKAN_LOGIN_PAGE_PATH, MIKAN_LOGIN_PAGE_SEARCH}, models::credential_3rd::{self, Credential3rdType}, }; -#[derive(Default, Clone, Deserialize, Serialize)] -pub struct MikanCredentialForm { - pub password: String, - pub username: String, - pub user_agent: String, -} - -impl Debug for MikanCredentialForm { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MikanCredentialForm") - .field("username", &String::from("[secrecy]")) - .field("password", &String::from("[secrecy]")) - .field("user_agent", &String::from("[secrecy]")) - .finish() - } -} #[derive(Debug)] pub struct MikanClient { @@ -154,6 +137,28 @@ impl MikanClient { } } + pub async fn save_credential( + &self, + ctx: Arc, + subscriber_id: i32, + credential_form: MikanCredentialForm, + ) -> RecorderResult { + let db = ctx.db(); + let am = credential_3rd::ActiveModel { + username: Set(Some(credential_form.username)), + password: Set(Some(credential_form.password)), + user_agent: Set(Some(credential_form.user_agent)), + credential_type: Set(Credential3rdType::Mikan), + subscriber_id: Set(subscriber_id), + ..Default::default() + } + .try_encrypt(ctx.clone()) + .await?; + + let credential: credential_3rd::Model = am.save(db).await?.try_into_model()?; + Ok(credential) + } + pub async fn fork_with_credential( &self, ctx: Arc, diff --git a/apps/recorder/src/extract/mikan/credential.rs b/apps/recorder/src/extract/mikan/credential.rs new file mode 100644 index 0000000..31053ae --- /dev/null +++ b/apps/recorder/src/extract/mikan/credential.rs @@ -0,0 +1,20 @@ +use std::fmt::Debug; + +use serde::{Deserialize, Serialize}; + +#[derive(Default, Clone, Deserialize, Serialize)] +pub struct MikanCredentialForm { + pub password: String, + pub username: String, + pub user_agent: String, +} + +impl Debug for MikanCredentialForm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MikanCredentialForm") + .field("username", &String::from("[secrecy]")) + .field("password", &String::from("[secrecy]")) + .field("user_agent", &String::from("[secrecy]")) + .finish() + } +} diff --git a/apps/recorder/src/extract/mikan/mod.rs b/apps/recorder/src/extract/mikan/mod.rs index cb82374..8715ea6 100644 --- a/apps/recorder/src/extract/mikan/mod.rs +++ b/apps/recorder/src/extract/mikan/mod.rs @@ -1,15 +1,17 @@ mod client; mod config; mod constants; +mod credential; mod rss; mod web; -pub use client::{MikanClient, MikanCredentialForm}; +pub use client::MikanClient; pub use config::MikanConfig; pub use constants::{ MIKAN_ACCOUNT_MANAGE_PAGE_PATH, MIKAN_LOGIN_PAGE_PATH, MIKAN_LOGIN_PAGE_SEARCH, MIKAN_POSTER_BUCKET_KEY, MIKAN_UNKNOWN_FANSUB_ID, MIKAN_UNKNOWN_FANSUB_NAME, }; +pub use credential::MikanCredentialForm; pub use rss::{ MikanBangumiIndexRssChannel, MikanBangumiRssChannel, MikanBangumiRssUrlMeta, MikanRssChannel, MikanRssItem, MikanSubscriberAggregationRssUrlMeta, MikanSubscriberStreamRssChannel, diff --git a/apps/recorder/src/extract/mikan/web.rs b/apps/recorder/src/extract/mikan/web.rs index be8535b..7e9792a 100644 --- a/apps/recorder/src/extract/mikan/web.rs +++ b/apps/recorder/src/extract/mikan/web.rs @@ -704,21 +704,25 @@ mod test { #![allow(unused_variables)] use std::fs; + use fetch::get_random_ua; use rstest::{fixture, rstest}; use tracing::Level; use url::Url; use zune_image::{codecs::ImageFormat, image::Image}; use super::*; - use crate::test_utils::{ - app::UnitTestAppContext, database::build_testing_database_service, - mikan::build_testing_mikan_client, storage::build_testing_storage_service, - tracing::try_init_testing_tracing, + use crate::{ + extract::mikan::MikanCredentialForm, + test_utils::{ + app::UnitTestAppContext, crypto::build_testing_crypto_service, + database::build_testing_database_service, mikan::build_testing_mikan_client, + storage::build_testing_storage_service, tracing::try_init_testing_tracing, + }, }; #[fixture] fn before_each() { - try_init_testing_tracing(Level::INFO); + try_init_testing_tracing(Level::DEBUG); } #[rstest] @@ -853,7 +857,7 @@ mod test { #[rstest] #[test] - fn ttest_extract_mikan_bangumi_meta_from_expand_subscribed_fragment( + fn test_extract_mikan_bangumi_meta_from_expand_subscribed_fragment( before_each: (), ) -> RecorderResult<()> { let origin_poster_src = @@ -934,9 +938,11 @@ mod test { let app_ctx = { let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?; let db_service = build_testing_database_service().await?; + let crypto_service = build_testing_crypto_service().await?; let app_ctx = UnitTestAppContext::builder() .mikan(mikan_client) .db(db_service) + .crypto(crypto_service) .build(); Arc::new(app_ctx) @@ -944,6 +950,18 @@ mod test { let mikan_client = app_ctx.mikan(); + let credential = mikan_client + .save_credential( + app_ctx.clone(), + 1, + MikanCredentialForm { + username: String::from("test_username"), + password: String::from("test_password"), + user_agent: get_random_ua().to_string(), + }, + ) + .await?; + let mikan_season_flow_url = build_mikan_season_flow_url(mikan_base_url, 2025, MikanSeasonStr::Spring); @@ -951,7 +969,7 @@ mod test { mikan_client, app_ctx.clone(), mikan_season_flow_url, - 1, + credential.id, ) .await?; diff --git a/apps/recorder/src/models/credential_3rd.rs b/apps/recorder/src/models/credential_3rd.rs index 2def4ea..68b4c1a 100644 --- a/apps/recorder/src/models/credential_3rd.rs +++ b/apps/recorder/src/models/credential_3rd.rs @@ -24,7 +24,7 @@ pub enum Credential3rdType { } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, DeriveEntityModel)] -#[sea_orm(table_name = "credential_3rd")] +#[sea_orm(table_name = "credential3rd")] pub struct Model { #[sea_orm(default_expr = "Expr::current_timestamp()")] pub created_at: DateTimeUtc, diff --git a/apps/recorder/src/test_utils/crypto.rs b/apps/recorder/src/test_utils/crypto.rs new file mode 100644 index 0000000..cc86db2 --- /dev/null +++ b/apps/recorder/src/test_utils/crypto.rs @@ -0,0 +1,9 @@ +use crate::{ + crypto::{CryptoConfig, CryptoService}, + errors::RecorderResult, +}; + +pub async fn build_testing_crypto_service() -> RecorderResult { + let crypto = CryptoService::from_config(CryptoConfig {}).await?; + Ok(crypto) +} diff --git a/apps/recorder/src/test_utils/database.rs b/apps/recorder/src/test_utils/database.rs index 69b0880..e683e26 100644 --- a/apps/recorder/src/test_utils/database.rs +++ b/apps/recorder/src/test_utils/database.rs @@ -5,7 +5,7 @@ use crate::{ #[cfg(feature = "testcontainers")] pub async fn build_testing_database_service() -> RecorderResult { - use testcontainers::runners::AsyncRunner; + use testcontainers::{ImageExt, runners::AsyncRunner}; use testcontainers_ext::{ImageDefaultLogConsumerExt, ImagePruneExistedLabelExt}; use testcontainers_modules::postgres::Postgres; @@ -13,6 +13,7 @@ pub async fn build_testing_database_service() -> RecorderResult .with_db_name("konobangu") .with_user("konobangu") .with_password("konobangu") + .with_tag("17-alpine") .with_default_log_consumer() .with_prune_existed_label(env!("CARGO_PKG_NAME"), "postgres", true, true) .await?; diff --git a/apps/recorder/src/test_utils/mod.rs b/apps/recorder/src/test_utils/mod.rs index f5c8717..2feadbe 100644 --- a/apps/recorder/src/test_utils/mod.rs +++ b/apps/recorder/src/test_utils/mod.rs @@ -1,4 +1,5 @@ pub mod app; +pub mod crypto; pub mod database; pub mod mikan; pub mod storage; diff --git a/apps/recorder/src/test_utils/tracing.rs b/apps/recorder/src/test_utils/tracing.rs index 210055b..058e68c 100644 --- a/apps/recorder/src/test_utils/tracing.rs +++ b/apps/recorder/src/test_utils/tracing.rs @@ -5,6 +5,7 @@ pub fn try_init_testing_tracing(level: Level) { let crate_name = env!("CARGO_PKG_NAME"); let level = level.as_str().to_lowercase(); let filter = EnvFilter::new(format!("{crate_name}[]={level}")) - .add_directive(format!("mockito[]={level}").parse().unwrap()); + .add_directive(format!("mockito[]={level}").parse().unwrap()) + .add_directive(format!("sqlx[]={level}").parse().unwrap()); let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init(); } diff --git a/justfile b/justfile index 1a39127..e6f3da0 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,13 @@ set windows-shell := ["pwsh.exe", "-c"] set dotenv-load := true prepare-dev-recorder: - cargo install sea-orm-cli watchexec cargo-llvm-cov cargo-nextest + cargo install sea-orm-cli cargo-llvm-cov cargo-nextest + # install watchexec + +prepare-dev-testcontainers: + docker pull linuxserver/qbittorrent:latest + docker pull ghcr.io/dumtruck/konobangu-testing-torrents:latest + docker pull postgres:17-alpine dev-webui: pnpm run --filter=webui dev diff --git a/packages/fetch/src/client/core.rs b/packages/fetch/src/client/core.rs index 94d318d..f7dacbf 100644 --- a/packages/fetch/src/client/core.rs +++ b/packages/fetch/src/client/core.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; use serde_with::serde_as; use snafu::Snafu; -use crate::get_random_mobile_ua; +use crate::get_random_ua; pub struct RateLimiterMiddleware { rate_limiter: RateLimiter, @@ -176,7 +176,7 @@ impl HttpClient { config .user_agent .as_deref() - .unwrap_or_else(|| get_random_mobile_ua()), + .unwrap_or_else(|| get_random_ua()), ); #[cfg(not(target_arch = "wasm32"))] @@ -291,7 +291,7 @@ impl HttpClient { self.config .user_agent .as_deref() - .unwrap_or_else(|| get_random_mobile_ua()), + .unwrap_or_else(|| get_random_ua()), ); #[cfg(not(target_arch = "wasm32"))] diff --git a/packages/fetch/src/core.rs b/packages/fetch/src/core.rs index c95432e..58340c9 100644 --- a/packages/fetch/src/core.rs +++ b/packages/fetch/src/core.rs @@ -5,7 +5,7 @@ lazy_static! { serde_json::from_str::>(include_str!("./ua.json")).unwrap(); } -pub fn get_random_mobile_ua() -> &'static str { +pub fn get_random_ua() -> &'static str { DEFAULT_HTTP_CLIENT_USER_AGENT[fastrand::usize(0..DEFAULT_HTTP_CLIENT_USER_AGENT.len())] .as_str() } diff --git a/packages/fetch/src/lib.rs b/packages/fetch/src/lib.rs index 220a5e1..7127cd7 100644 --- a/packages/fetch/src/lib.rs +++ b/packages/fetch/src/lib.rs @@ -6,7 +6,7 @@ pub mod html; pub mod image; pub mod test_util; -pub use core::get_random_mobile_ua; +pub use core::get_random_ua; pub use bytes::fetch_bytes; pub use client::{HttpClient, HttpClientConfig, HttpClientError, HttpClientTrait};