dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split modules

Browse Source
This commit is contained in:
master
2025-04-08 02:12:06 +08:00
parent 376d2b28d3
commit 2686fa1d76
94 changed files with 1125 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
apps/recorder/src/extract/mikan/client.rs
View File

@@ -1,15 +1,11 @@
use std::{fmt::Debug, ops::Deref};
use reqwest_middleware::ClientWithMiddleware;
use fetch::{FetchError, HttpClient, HttpClientTrait, client::HttpClientCookiesAuth};
use serde::{Deserialize, Serialize};
use url::Url;
use super::MikanConfig;
use crate::{
errors::app_error::RError,
fetch::{HttpClient, HttpClientTrait, client::HttpClientCookiesAuth},
};
use crate::errors::RecorderError;
#[derive(Default, Clone, Deserialize, Serialize)]
pub struct MikanAuthSecrecy {
pub cookie: String,
@@ -26,8 +22,10 @@ impl Debug for MikanAuthSecrecy {
}
impl MikanAuthSecrecy {
pub fn into_cookie_auth(self, url: &Url) -> Result<HttpClientCookiesAuth, RError> {
pub fn into_cookie_auth(self, url: &Url) -> Result<HttpClientCookiesAuth, RecorderError> {
HttpClientCookiesAuth::from_cookies(&self.cookie, url, self.user_agent)
.map_err(FetchError::from)
.map_err(RecorderError::from)
}
}
@@ -38,7 +36,7 @@ pub struct MikanClient {
}
impl MikanClient {
pub async fn from_config(config: MikanConfig) -> Result<Self, RError> {
pub async fn from_config(config: MikanConfig) -> Result<Self, RecorderError> {
let http_client = HttpClient::from_config(config.http_client)?;
let base_url = config.base_url;
Ok(Self {
@@ -47,7 +45,7 @@ impl MikanClient {
})
}
pub fn fork_with_auth(&self, secrecy: Option<MikanAuthSecrecy>) -> Result<Self, RError> {
pub fn fork_with_auth(&self, secrecy: Option<MikanAuthSecrecy>) -> Result<Self, RecorderError> {
let mut fork = self.http_client.fork();
if let Some(secrecy) = secrecy {
@@ -71,10 +69,10 @@ impl MikanClient {
}
impl Deref for MikanClient {
type Target = ClientWithMiddleware;
type Target = fetch::reqwest_middleware::ClientWithMiddleware;
fn deref(&self) -> &Self::Target {
self.http_client.deref()
&self.http_client
}
}

3
apps/recorder/src/extract/mikan/config.rs
View File

@@ -1,8 +1,7 @@
use fetch::HttpClientConfig;
use serde::{Deserialize, Serialize};
use url::Url;
use crate::fetch::HttpClientConfig;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MikanConfig {
pub http_client: HttpClientConfig,

63
apps/recorder/src/extract/mikan/rss_extract.rs
View File

@@ -1,22 +1,19 @@
use std::borrow::Cow;
use chrono::DateTime;
use downloader::bittorrent::defs::BITTORRENT_MIME_TYPE;
use fetch::{FetchError, IntoUrl, bytes::fetch_bytes};
use itertools::Itertools;
use reqwest::IntoUrl;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use url::Url;
use crate::{
errors::app_error::{RError, RResult},
extract::{
bittorrent::BITTORRENT_MIME_TYPE,
mikan::{
MikanClient,
web_extract::{MikanEpisodeHomepage, extract_mikan_episode_id_from_homepage},
},
errors::app_error::{RecorderError, RecorderResult},
extract::mikan::{
MikanClient,
web_extract::{MikanEpisodeHomepage, extract_mikan_episode_id_from_homepage},
},
fetch::bytes::fetch_bytes,
};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -102,28 +99,28 @@ impl MikanRssChannel {
}
impl TryFrom<rss::Item> for MikanRssItem {
type Error = RError;
type Error = RecorderError;
fn try_from(item: rss::Item) -> Result<Self, Self::Error> {
let enclosure = item
.enclosure
.ok_or_else(|| RError::from_mikan_rss_invalid_field(Cow::Borrowed("enclosure")))?;
let enclosure = item.enclosure.ok_or_else(|| {
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("enclosure"))
})?;
let mime_type = enclosure.mime_type;
if mime_type != BITTORRENT_MIME_TYPE {
return Err(RError::MimeError {
return Err(RecorderError::MimeError {
expected: String::from(BITTORRENT_MIME_TYPE),
found: mime_type.to_string(),
desc: String::from("MikanRssItem"),
});
}
let title = item
.title
.ok_or_else(|| RError::from_mikan_rss_invalid_field(Cow::Borrowed("title:title")))?;
let title = item.title.ok_or_else(|| {
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("title:title"))
})?;
let enclosure_url = Url::parse(&enclosure.url).map_err(|err| {
RError::from_mikan_rss_invalid_field_and_source(
RecorderError::from_mikan_rss_invalid_field_and_source(
"enclosure_url:enclosure.link".into(),
err,
)
@@ -132,12 +129,14 @@ impl TryFrom<rss::Item> for MikanRssItem {
let homepage = item
.link
.and_then(|link| Url::parse(&link).ok())
.ok_or_else(|| RError::from_mikan_rss_invalid_field(Cow::Borrowed("homepage:link")))?;
.ok_or_else(|| {
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("homepage:link"))
})?;
let MikanEpisodeHomepage {
mikan_episode_id, ..
} = extract_mikan_episode_id_from_homepage(&homepage).ok_or_else(|| {
RError::from_mikan_rss_invalid_field(Cow::Borrowed("mikan_episode_id"))
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("mikan_episode_id"))
})?;
Ok(MikanRssItem {
@@ -170,8 +169,8 @@ pub fn build_mikan_bangumi_rss_link(
mikan_base_url: impl IntoUrl,
mikan_bangumi_id: &str,
mikan_fansub_id: Option<&str>,
) -> RResult<Url> {
let mut url = mikan_base_url.into_url()?;
) -> RecorderResult<Url> {
let mut url = mikan_base_url.into_url().map_err(FetchError::from)?;
url.set_path("/RSS/Bangumi");
url.query_pairs_mut()
.append_pair("bangumiId", mikan_bangumi_id);
@@ -185,7 +184,7 @@ pub fn build_mikan_bangumi_rss_link(
pub fn build_mikan_subscriber_aggregation_rss_link(
mikan_base_url: &str,
mikan_aggregation_id: &str,
) -> RResult<Url> {
) -> RecorderResult<Url> {
let mut url = Url::parse(mikan_base_url)?;
url.set_path("/RSS/MyBangumi");
url.query_pairs_mut()
@@ -227,7 +226,7 @@ pub fn extract_mikan_subscriber_aggregation_id_from_rss_link(
pub async fn extract_mikan_rss_channel_from_rss_link(
http_client: &MikanClient,
channel_rss_link: impl IntoUrl,
) -> RResult<MikanRssChannel> {
) -> RecorderResult<MikanRssChannel> {
let bytes = fetch_bytes(http_client, channel_rss_link.as_str()).await?;
let channel = rss::Channel::read_from(&bytes[..])?;
@@ -326,7 +325,7 @@ pub async fn extract_mikan_rss_channel_from_rss_link(
},
))
} else {
Err(RError::MikanRssInvalidFormatError).inspect_err(|error| {
Err(RecorderError::MikanRssInvalidFormatError).inspect_err(|error| {
tracing::warn!(error = %error);
})
}
@@ -336,24 +335,22 @@ pub async fn extract_mikan_rss_channel_from_rss_link(
mod tests {
use std::assert_matches::assert_matches;
use downloader::bittorrent::BITTORRENT_MIME_TYPE;
use rstest::rstest;
use url::Url;
use crate::{
errors::app_error::RResult,
extract::{
bittorrent::BITTORRENT_MIME_TYPE,
mikan::{
MikanBangumiAggregationRssChannel, MikanBangumiRssChannel, MikanRssChannel,
extract_mikan_rss_channel_from_rss_link,
},
errors::RecorderResult,
extract::mikan::{
MikanBangumiAggregationRssChannel, MikanBangumiRssChannel, MikanRssChannel,
extract_mikan_rss_channel_from_rss_link,
},
test_utils::mikan::build_testing_mikan_client,
};
#[rstest]
#[tokio::test]
async fn test_parse_mikan_rss_channel_from_rss_link() -> RResult<()> {
async fn test_parse_mikan_rss_channel_from_rss_link() -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;

48
apps/recorder/src/extract/mikan/web_extract.rs
View File

@@ -2,6 +2,7 @@ use std::{borrow::Cow, sync::Arc};
use async_stream::try_stream;
use bytes::Bytes;
use fetch::{html::fetch_html, image::fetch_image};
use futures::Stream;
use itertools::Itertools;
use scraper::{Html, Selector};
@@ -15,12 +16,11 @@ use super::{
};
use crate::{
app::AppContextTrait,
errors::app_error::{RError, RResult},
errors::app_error::{RecorderResult, RecorderError},
extract::{
html::{extract_background_image_src_from_style_attr, extract_inner_text_from_element_ref},
media::extract_image_src_from_str,
},
fetch::{html::fetch_html, image::fetch_image},
storage::StorageContentCategory,
};
@@ -115,7 +115,7 @@ pub fn extract_mikan_episode_id_from_homepage(url: &Url) -> Option<MikanEpisodeH
pub async fn extract_mikan_poster_meta_from_src(
http_client: &MikanClient,
origin_poster_src_url: Url,
) -> Result<MikanBangumiPosterMeta, RError> {
) -> Result<MikanBangumiPosterMeta, RecorderError> {
let poster_data = fetch_image(http_client, origin_poster_src_url.clone()).await?;
Ok(MikanBangumiPosterMeta {
origin_poster_src: origin_poster_src_url,
@@ -128,7 +128,7 @@ pub async fn extract_mikan_bangumi_poster_meta_from_src_with_cache(
ctx: &dyn AppContextTrait,
origin_poster_src_url: Url,
subscriber_id: i32,
) -> RResult<MikanBangumiPosterMeta> {
) -> RecorderResult<MikanBangumiPosterMeta> {
let dal_client = ctx.storage();
let mikan_client = ctx.mikan();
if let Some(poster_src) = dal_client
@@ -170,7 +170,7 @@ pub async fn extract_mikan_bangumi_poster_meta_from_src_with_cache(
pub async fn extract_mikan_episode_meta_from_episode_homepage(
http_client: &MikanClient,
mikan_episode_homepage_url: Url,
) -> Result<MikanEpisodeMeta, RError> {
) -> Result<MikanEpisodeMeta, RecorderError> {
let mikan_base_url = Url::parse(&mikan_episode_homepage_url.origin().unicode_serialization())?;
let content = fetch_html(http_client, mikan_episode_homepage_url.as_str()).await?;
@@ -186,7 +186,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
.select(bangumi_title_selector)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -201,18 +201,22 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
.and_then(|el| el.value().attr("href"))
.and_then(|s| mikan_episode_homepage_url.join(s).ok())
.and_then(|rss_link_url| extract_mikan_bangumi_id_from_rss_link(&rss_link_url))
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id")))
.ok_or_else(|| {
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
let mikan_fansub_id = mikan_fansub_id
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_fansub_id")))
.ok_or_else(|| {
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_fansub_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
let episode_title = html
.select(&Selector::parse("title").unwrap())
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("episode_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("episode_title")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -220,7 +224,9 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
let MikanEpisodeHomepage {
mikan_episode_id, ..
} = extract_mikan_episode_id_from_homepage(&mikan_episode_homepage_url)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_episode_id")))
.ok_or_else(|| {
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_episode_id"))
})
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -232,7 +238,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("fansub_name")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("fansub_name")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -275,7 +281,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
http_client: &MikanClient,
mikan_bangumi_homepage_url: Url,
) -> Result<MikanBangumiMeta, RError> {
) -> Result<MikanBangumiMeta, RecorderError> {
let mikan_base_url = Url::parse(&mikan_bangumi_homepage_url.origin().unicode_serialization())?;
let content = fetch_html(http_client, mikan_bangumi_homepage_url.as_str()).await?;
let html = Html::parse_document(&content);
@@ -289,7 +295,7 @@ pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
.select(bangumi_title_selector)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.inspect_err(|error| tracing::warn!(error = %error))?;
let mikan_bangumi_id = html
@@ -303,7 +309,9 @@ pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
mikan_bangumi_id, ..
}| mikan_bangumi_id,
)
.ok_or_else(|| RError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id")))
.ok_or_else(|| {
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
let origin_poster_src = html.select(bangumi_poster_selector).next().and_then(|el| {
@@ -353,8 +361,8 @@ pub fn extract_mikan_bangumis_meta_from_my_bangumi_page(
context: Arc<dyn AppContextTrait>,
my_bangumi_page_url: Url,
auth_secrecy: Option<MikanAuthSecrecy>,
history: &[Arc<RResult<MikanBangumiMeta>>],
) -> impl Stream<Item = RResult<MikanBangumiMeta>> {
history: &[Arc<RecorderResult<MikanBangumiMeta>>],
) -> impl Stream<Item = RecorderResult<MikanBangumiMeta>> {
try_stream! {
let http_client = &context.mikan().fork_with_auth(auth_secrecy.clone())?;
@@ -511,7 +519,7 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_extract_mikan_poster_from_src(before_each: ()) -> RResult<()> {
async fn test_extract_mikan_poster_from_src(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
@@ -542,7 +550,7 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_extract_mikan_episode(before_each: ()) -> RResult<()> {
async fn test_extract_mikan_episode(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
@@ -582,7 +590,7 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_extract_mikan_bangumi_meta_from_bangumi_homepage(before_each: ()) -> RResult<()> {
async fn test_extract_mikan_bangumi_meta_from_bangumi_homepage(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
@@ -619,7 +627,7 @@ mod test {
#[rstest]
#[tokio::test]
async fn test_extract_mikan_bangumis_meta_from_my_bangumi_page(before_each: ()) -> RResult<()> {
async fn test_extract_mikan_bangumis_meta_from_my_bangumi_page(before_each: ()) -> RecorderResult<()> {
let mut mikan_server = mockito::Server::new_async().await;
let mikan_base_url = Url::parse(&mikan_server.url())?;