dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fix testsuite

Browse Source
This commit is contained in:
master
2025-05-26 02:44:46 +08:00
parent 313b1bf1ba
commit 22a2ce0559
1001 changed files with 299176 additions and 5417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
apps/recorder/src/extract/mikan/client.rs
View File

@@ -2,6 +2,7 @@ use std::{fmt::Debug, ops::Deref, sync::Arc};
use fetch::{HttpClient, HttpClientTrait};
use maplit::hashmap;
use scraper::{Html, Selector};
use sea_orm::{
ActiveModelTrait, ActiveValue::Set, ColumnTrait, DbErr, EntityTrait, QueryFilter, TryIntoModel,
};
@@ -68,50 +69,44 @@ impl MikanClient {
message: "mikan login failed, credential required".to_string(),
source: None.into(),
})?;
let login_page_url = {
let mut u = self.base_url.join(MIKAN_LOGIN_PAGE_PATH)?;
u.set_query(Some(MIKAN_LOGIN_PAGE_SEARCH));
u
};
// access login page to get antiforgery cookie
self.http_client
.get(login_page_url.clone())
.send()
.await
.map_err(|error| RecorderError::Credential3rdError {
message: "failed to get mikan login page".to_string(),
source: OptDynErr::some_boxed(error),
})?;
let antiforgery_token = {
// access login page to get antiforgery cookie
let login_page_html = self
.http_client
.get(login_page_url.clone())
.send()
.await
.map_err(|error| RecorderError::Credential3rdError {
message: "failed to get mikan login page".to_string(),
source: OptDynErr::some_boxed(error),
})?
.text()
.await?;
let antiforgery_cookie = {
let cookie_store_lock = self.http_client.cookie_store.clone().ok_or_else(|| {
RecorderError::Credential3rdError {
message: "failed to get cookie store".to_string(),
let login_page_html = Html::parse_document(&login_page_html);
let antiforgery_selector =
Selector::parse("input[name='__RequestVerificationToken']").unwrap();
login_page_html
.select(&antiforgery_selector)
.next()
.and_then(|element| element.value().attr("value").map(|value| value.to_string()))
.ok_or_else(|| RecorderError::Credential3rdError {
message: "mikan login failed, failed to get antiforgery token".to_string(),
source: None.into(),
}
})?;
let cookie_store =
cookie_store_lock
.read()
.map_err(|_| RecorderError::Credential3rdError {
message: "failed to read cookie store".to_string(),
source: None.into(),
})?;
cookie_store
.matches(&login_page_url)
.iter()
.find(|cookie| cookie.name().starts_with(".AspNetCore.Antiforgery."))
.map(|cookie| cookie.value().to_string())
}
.ok_or_else(|| RecorderError::Credential3rdError {
message: "mikan login failed, failed to get antiforgery cookie".to_string(),
source: None.into(),
})?;
})
}?;
let login_post_form = hashmap! {
"__RequestVerificationToken".to_string() => antiforgery_cookie,
"__RequestVerificationToken".to_string() => antiforgery_token,
"UserName".to_string() => userpass_credential.username.clone(),
"Password".to_string() => userpass_credential.password.clone(),
"RememberMe".to_string() => "true".to_string(),
@@ -185,12 +180,33 @@ impl MikanClient {
}
pub async fn fork_with_credential(
&self,
userpass_credential: UserPassCredential,
) -> RecorderResult<Self> {
let mut fork = self
.http_client
.fork()
.attach_cookies(userpass_credential.cookies.as_deref())?;
if let Some(user_agent) = userpass_credential.user_agent.as_ref() {
fork = fork.attach_user_agent(user_agent);
}
let userpass_credential_opt = Some(userpass_credential);
Ok(Self {
http_client: HttpClient::from_fork(fork)?,
base_url: self.base_url.clone(),
origin_url: self.origin_url.clone(),
userpass_credential: userpass_credential_opt,
})
}
pub async fn fork_with_credential_id(
&self,
ctx: Arc<dyn AppContextTrait>,
credential_id: i32,
) -> RecorderResult<Self> {
let mut fork = self.http_client.fork();
let credential = credential_3rd::Model::find_by_id(ctx.clone(), credential_id).await?;
if let Some(credential) = credential {
if credential.credential_type != Credential3rdType::Mikan {
@@ -203,20 +219,7 @@ impl MikanClient {
let userpass_credential: UserPassCredential =
credential.try_into_userpass_credential(ctx)?;
fork = fork.attach_cookies(userpass_credential.cookies.as_deref())?;
if let Some(user_agent) = userpass_credential.user_agent.as_ref() {
fork = fork.attach_user_agent(user_agent);
}
let userpass_credential_opt = Some(userpass_credential);
Ok(Self {
http_client: HttpClient::from_fork(fork)?,
base_url: self.base_url.clone(),
origin_url: self.origin_url.clone(),
userpass_credential: userpass_credential_opt,
})
self.fork_with_credential(userpass_credential).await
} else {
Err(RecorderError::from_db_record_not_found(
DbErr::RecordNotFound(format!("credential={credential_id} not found")),
@@ -319,7 +322,7 @@ mod tests {
);
let mikan_client = mikan_client
.fork_with_credential(app_ctx.clone(), credential_model.id)
.fork_with_credential_id(app_ctx.clone(), credential_model.id)
.await?;
mikan_client.login().await?;

9
apps/recorder/src/extract/mikan/constants.rs
View File

@@ -5,4 +5,13 @@ pub const MIKAN_LOGIN_PAGE_PATH: &str = "/Account/Login";
pub const MIKAN_LOGIN_PAGE_SEARCH: &str = "ReturnUrl=%2F";
pub const MIKAN_ACCOUNT_MANAGE_PAGE_PATH: &str = "/Account/Manage";
pub const MIKAN_SEASON_FLOW_PAGE_PATH: &str = "/Home/BangumiCoverFlow";
pub const MIKAN_BANGUMI_HOMEPAGE_PATH: &str = "/Home/Bangumi";
pub const MIKAN_BANGUMI_EXPAND_SUBSCRIBED_PAGE_PATH: &str = "/Home/ExpandBangumi";
pub const MIKAN_EPISODE_HOMEPAGE_PATH: &str = "/Home/Episode";
pub const MIKAN_BANGUMI_POSTER_PATH: &str = "/images/Bangumi";
pub const MIKAN_EPISODE_TORRENT_PATH: &str = "/Download";
pub const MIKAN_SUBSCRIBER_SUBSCRIPTION_RSS_PATH: &str = "/RSS/MyBangumi";
pub const MIKAN_BANGUMI_RSS_PATH: &str = "/RSS/Bangumi";
pub const MIKAN_BANGUMI_ID_QUERY_KEY: &str = "bangumiId";
pub const MIKAN_FANSUB_ID_QUERY_KEY: &str = "subgroupid";
pub const MIKAN_SUBSCRIBER_SUBSCRIPTION_TOKEN_QUERY_KEY: &str = "token";

8
apps/recorder/src/extract/mikan/mod.rs
View File

@@ -9,8 +9,12 @@ pub use client::MikanClient;
pub use config::MikanConfig;
pub use constants::{
MIKAN_ACCOUNT_MANAGE_PAGE_PATH, MIKAN_BANGUMI_EXPAND_SUBSCRIBED_PAGE_PATH,
MIKAN_LOGIN_PAGE_PATH, MIKAN_LOGIN_PAGE_SEARCH, MIKAN_POSTER_BUCKET_KEY,
MIKAN_SEASON_FLOW_PAGE_PATH, MIKAN_UNKNOWN_FANSUB_ID, MIKAN_UNKNOWN_FANSUB_NAME,
MIKAN_BANGUMI_HOMEPAGE_PATH, MIKAN_BANGUMI_ID_QUERY_KEY, MIKAN_BANGUMI_POSTER_PATH,
MIKAN_BANGUMI_RSS_PATH, MIKAN_EPISODE_HOMEPAGE_PATH, MIKAN_EPISODE_TORRENT_PATH,
MIKAN_FANSUB_ID_QUERY_KEY, MIKAN_LOGIN_PAGE_PATH, MIKAN_LOGIN_PAGE_SEARCH,
MIKAN_POSTER_BUCKET_KEY, MIKAN_SEASON_FLOW_PAGE_PATH, MIKAN_SUBSCRIBER_SUBSCRIPTION_RSS_PATH,
MIKAN_SUBSCRIBER_SUBSCRIPTION_TOKEN_QUERY_KEY, MIKAN_UNKNOWN_FANSUB_ID,
MIKAN_UNKNOWN_FANSUB_NAME,
};
pub use credential::MikanCredentialForm;
pub use subscription::{

141
apps/recorder/src/extract/mikan/web.rs
View File

@@ -20,8 +20,11 @@ use crate::{
html::{extract_background_image_src_from_style_attr, extract_inner_text_from_element_ref},
media::extract_image_src_from_str,
mikan::{
MIKAN_BANGUMI_EXPAND_SUBSCRIBED_PAGE_PATH, MIKAN_POSTER_BUCKET_KEY,
MIKAN_SEASON_FLOW_PAGE_PATH, MikanClient,
MIKAN_BANGUMI_EXPAND_SUBSCRIBED_PAGE_PATH, MIKAN_BANGUMI_HOMEPAGE_PATH,
MIKAN_BANGUMI_ID_QUERY_KEY, MIKAN_BANGUMI_POSTER_PATH, MIKAN_BANGUMI_RSS_PATH,
MIKAN_EPISODE_HOMEPAGE_PATH, MIKAN_FANSUB_ID_QUERY_KEY, MIKAN_POSTER_BUCKET_KEY,
MIKAN_SEASON_FLOW_PAGE_PATH, MIKAN_SUBSCRIBER_SUBSCRIPTION_RSS_PATH,
MIKAN_SUBSCRIBER_SUBSCRIPTION_TOKEN_QUERY_KEY, MikanClient,
},
},
storage::{StorageContentCategory, StorageServiceTrait},
@@ -101,12 +104,12 @@ pub struct MikanSubscriberSubscriptionRssUrlMeta {
impl MikanSubscriberSubscriptionRssUrlMeta {
pub fn from_rss_url(url: &Url) -> Option<Self> {
if url.path() == "/RSS/MyBangumi" {
url.query_pairs().find(|(k, _)| k == "token").map(|(_, v)| {
MikanSubscriberSubscriptionRssUrlMeta {
if url.path() == MIKAN_SUBSCRIBER_SUBSCRIPTION_RSS_PATH {
url.query_pairs()
.find(|(k, _)| k == MIKAN_SUBSCRIBER_SUBSCRIPTION_TOKEN_QUERY_KEY)
.map(|(_, v)| MikanSubscriberSubscriptionRssUrlMeta {
mikan_subscription_token: v.to_string(),
}
})
})
} else {
None
}
@@ -122,9 +125,11 @@ pub fn build_mikan_subscriber_subscription_rss_url(
mikan_subscription_token: &str,
) -> Url {
let mut url = mikan_base_url;
url.set_path("/RSS/MyBangumi");
url.query_pairs_mut()
.append_pair("token", mikan_subscription_token);
url.set_path(MIKAN_SUBSCRIBER_SUBSCRIPTION_RSS_PATH);
url.query_pairs_mut().append_pair(
MIKAN_SUBSCRIBER_SUBSCRIPTION_TOKEN_QUERY_KEY,
mikan_subscription_token,
);
url
}
@@ -224,8 +229,10 @@ pub struct MikanBangumiIndexHash {
impl MikanBangumiIndexHash {
pub fn from_homepage_url(url: &Url) -> Option<Self> {
if url.path().starts_with("/Home/Bangumi/") {
let mikan_bangumi_id = url.path().replace("/Home/Bangumi/", "");
if url.path().starts_with(MIKAN_BANGUMI_HOMEPAGE_PATH) {
let mikan_bangumi_id = url
.path()
.replace(&format!("{MIKAN_BANGUMI_HOMEPAGE_PATH}/"), "");
Some(Self { mikan_bangumi_id })
} else {
@@ -244,12 +251,12 @@ pub fn build_mikan_bangumi_subscription_rss_url(
mikan_fansub_id: Option<&str>,
) -> Url {
let mut url = mikan_base_url;
url.set_path("/RSS/Bangumi");
url.set_path(MIKAN_BANGUMI_RSS_PATH);
url.query_pairs_mut()
.append_pair("bangumiId", mikan_bangumi_id);
.append_pair(MIKAN_BANGUMI_ID_QUERY_KEY, mikan_bangumi_id);
if let Some(mikan_fansub_id) = mikan_fansub_id {
url.query_pairs_mut()
.append_pair("subgroupid", mikan_fansub_id);
.append_pair(MIKAN_FANSUB_ID_QUERY_KEY, mikan_fansub_id);
};
url
}
@@ -262,8 +269,10 @@ pub struct MikanBangumiHash {
impl MikanBangumiHash {
pub fn from_homepage_url(url: &Url) -> Option<Self> {
if url.path().starts_with("/Home/Bangumi/") {
let mikan_bangumi_id = url.path().replace("/Home/Bangumi/", "");
if url.path().starts_with(MIKAN_BANGUMI_HOMEPAGE_PATH) {
let mikan_bangumi_id = url
.path()
.replace(&format!("{MIKAN_BANGUMI_HOMEPAGE_PATH}/"), "");
let url_fragment = url.fragment()?;
@@ -277,13 +286,13 @@ impl MikanBangumiHash {
}
pub fn from_rss_url(url: &Url) -> Option<Self> {
if url.path() == "/RSS/Bangumi" {
if url.path() == MIKAN_BANGUMI_RSS_PATH {
if let (Some(mikan_fansub_id), Some(mikan_bangumi_id)) = (
url.query_pairs()
.find(|(k, _)| k == "subgroupid")
.find(|(k, _)| k == MIKAN_FANSUB_ID_QUERY_KEY)
.map(|(_, v)| v.to_string()),
url.query_pairs()
.find(|(k, _)| k == "bangumiId")
.find(|(k, _)| k == MIKAN_BANGUMI_ID_QUERY_KEY)
.map(|(_, v)| v.to_string()),
) {
Some(Self {
@@ -317,7 +326,7 @@ impl MikanBangumiHash {
pub fn build_mikan_episode_homepage_url(mikan_base_url: Url, mikan_episode_id: &str) -> Url {
let mut url = mikan_base_url;
url.set_path(&format!("/Home/Episode/{mikan_episode_id}"));
url.set_path(&format!("{MIKAN_EPISODE_HOMEPAGE_PATH}/{mikan_episode_id}"));
url
}
@@ -328,8 +337,10 @@ pub struct MikanEpisodeHash {
impl MikanEpisodeHash {
pub fn from_homepage_url(url: &Url) -> Option<Self> {
if url.path().starts_with("/Home/Episode/") {
let mikan_episode_id = url.path().replace("/Home/Episode/", "");
if url.path().starts_with(MIKAN_EPISODE_HOMEPAGE_PATH) {
let mikan_episode_id = url
.path()
.replace(&format!("{MIKAN_EPISODE_HOMEPAGE_PATH}/"), "");
Some(Self { mikan_episode_id })
} else {
None
@@ -416,7 +427,7 @@ pub fn build_mikan_bangumi_homepage_url(
mikan_fansub_id: Option<&str>,
) -> Url {
let mut url = mikan_base_url;
url.set_path(&format!("/Home/Bangumi/{mikan_bangumi_id}"));
url.set_path(&format!("{MIKAN_BANGUMI_HOMEPAGE_PATH}/{mikan_bangumi_id}"));
url.set_fragment(mikan_fansub_id);
url
}
@@ -715,7 +726,9 @@ pub async fn scrape_mikan_poster_meta_from_image_url(
StorageContentCategory::Image,
subscriber_id,
Some(MIKAN_POSTER_BUCKET_KEY),
&origin_poster_src_url.path().replace("/images/Bangumi/", ""),
&origin_poster_src_url
.path()
.replace(&format!("{MIKAN_BANGUMI_POSTER_PATH}/"), ""),
)
.await?
{
@@ -734,7 +747,9 @@ pub async fn scrape_mikan_poster_meta_from_image_url(
StorageContentCategory::Image,
subscriber_id,
Some(MIKAN_POSTER_BUCKET_KEY),
&origin_poster_src_url.path().replace("/images/Bangumi/", ""),
&origin_poster_src_url
.path()
.replace(&format!("{MIKAN_BANGUMI_POSTER_PATH}/"), ""),
poster_data,
)
.await?;
@@ -884,7 +899,7 @@ pub fn scrape_mikan_bangumi_meta_stream_from_season_flow_url(
) -> impl Stream<Item = RecorderResult<MikanBangumiMeta>> {
try_stream! {
let mikan_base_url = ctx.mikan().base_url().clone();
let mikan_client = ctx.mikan().fork_with_credential(ctx.clone(), credential_id).await?;
let mikan_client = ctx.mikan().fork_with_credential_id(ctx.clone(), credential_id).await?;
let content = fetch_html(&mikan_client, mikan_season_flow_url.clone()).await?;
@@ -971,7 +986,8 @@ mod test {
crypto::build_testing_crypto_service,
database::build_testing_database_service,
mikan::{
MikanMockServer, build_testing_mikan_client, build_testing_mikan_credential_form,
MikanMockServer, build_testing_mikan_client, build_testing_mikan_credential,
build_testing_mikan_credential_form,
},
storage::build_testing_storage_service,
tracing::try_init_testing_tracing,
@@ -986,22 +1002,19 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_scrape_mikan_poster_data_from_image_url(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mut mikan_server = MikanMockServer::new().await?;
let resources_mock = mikan_server.mock_resources_with_doppel();
let mikan_base_url = mikan_server.base_url().clone();
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
let bangumi_poster_url = mikan_base_url.join("/images/Bangumi/202309/5ce9fed1.jpg")?;
let bangumi_poster_mock = mikan_server
.mock("GET", bangumi_poster_url.path())
.with_body_from_file("tests/resources/mikan/Bangumi-202309-5ce9fed1.jpg")
.create_async()
.await;
let bgm_poster_data =
scrape_mikan_poster_data_from_image_url(&mikan_client, bangumi_poster_url).await?;
bangumi_poster_mock.expect(1);
resources_mock.shared_resource_mock.expect(1);
let image = Image::read(bgm_poster_data.to_vec(), Default::default());
assert!(
image.is_ok_and(|img| img
@@ -1017,20 +1030,19 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_scrape_mikan_poster_meta_from_image_url(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mut mikan_server = MikanMockServer::new().await?;
let mikan_base_url = mikan_server.base_url().clone();
let resources_mock = mikan_server.mock_resources_with_doppel();
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
let storage_service = build_testing_storage_service().await?;
let storage_operator = storage_service.get_operator()?;
let bangumi_poster_url = mikan_base_url.join("/images/Bangumi/202309/5ce9fed1.jpg")?;
let bangumi_poster_mock = mikan_server
.mock("GET", bangumi_poster_url.path())
.with_body_from_file("tests/resources/mikan/Bangumi-202309-5ce9fed1.jpg")
.create_async()
.await;
let bgm_poster = scrape_mikan_poster_meta_from_image_url(
&mikan_client,
&storage_service,
@@ -1039,7 +1051,7 @@ mod test {
)
.await?;
bangumi_poster_mock.expect(1);
resources_mock.shared_resource_mock.expect(1);
let storage_fullname = storage_service.get_fullname(
StorageContentCategory::Image,
@@ -1051,7 +1063,8 @@ mod test {
assert!(storage_operator.exists(storage_fullename_str).await?);
let expected_data = fs::read("tests/resources/mikan/Bangumi-202309-5ce9fed1.jpg")?;
let expected_data =
fs::read("tests/resources/mikan/doppel/images/Bangumi/202309/5ce9fed1.jpg")?;
let found_data = storage_operator.read(storage_fullename_str).await?.to_vec();
assert_eq!(expected_data, found_data);
@@ -1100,7 +1113,7 @@ mod test {
before_each: (),
) -> RecorderResult<()> {
let fragment_str =
fs::read_to_string("tests/resources/mikan/BangumiCoverFlow-2025-spring-noauth.html")?;
fs::read_to_string("tests/resources/mikan/BangumiCoverFlow-noauth.html")?;
let bangumi_index_meta_list =
extract_mikan_bangumi_index_meta_list_from_season_flow_fragment(
@@ -1114,12 +1127,27 @@ mod test {
}
#[rstest]
#[test]
fn test_extract_mikan_bangumi_meta_from_expand_subscribed_fragment(
#[tokio::test]
async fn test_extract_mikan_bangumi_meta_from_expand_subscribed_fragment(
before_each: (),
) -> RecorderResult<()> {
let mut mikan_server = MikanMockServer::new().await?;
let login_mock = mikan_server.mock_get_login_page();
let resources_mock = mikan_server.mock_resources_with_doppel();
let mikan_base_url = mikan_server.base_url().clone();
let mikan_client = build_testing_mikan_client(mikan_base_url.clone())
.await?
.fork_with_credential(build_testing_mikan_credential())
.await?;
mikan_client.login().await?;
let origin_poster_src =
Url::parse("https://mikanani.me/images/Bangumi/202504/076c1094.jpg")?;
let bangumi_index_meta = MikanBangumiIndexMeta {
homepage: Url::parse("https://mikanani.me/Home/Bangumi/3599")?,
origin_poster_src: Some(origin_poster_src.clone()),
@@ -1127,10 +1155,17 @@ mod test {
mikan_bangumi_id: "3599".to_string(),
};
let fragment_str = fs::read_to_string("tests/resources/mikan/ExpandBangumi-3599.html")?;
let fragment_str = fetch_html(
&mikan_client,
build_mikan_bangumi_expand_subscribed_url(
mikan_base_url.clone(),
&bangumi_index_meta.mikan_bangumi_id,
),
)
.await?;
let bangumi = extract_mikan_bangumi_meta_from_expand_subscribed_fragment(
&Html::parse_document(&fragment_str),
&Html::parse_fragment(&fragment_str),
bangumi_index_meta.clone(),
Url::parse("https://mikanani.me/")?,
)
@@ -1175,7 +1210,7 @@ mod test {
fs::read_to_string("tests/resources/mikan/ExpandBangumi-3599-noauth.html")?;
let bangumi = extract_mikan_bangumi_meta_from_expand_subscribed_fragment(
&Html::parse_document(&fragment_str),
&Html::parse_fragment(&fragment_str),
bangumi_index_meta.clone(),
Url::parse("https://mikanani.me/")?,
);