dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add mikan cookie support

Browse Source
This commit is contained in:
master
2025-02-26 05:31:28 +08:00
parent cac0d37e53
commit 6887b2a069
24 changed files with 1413 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
apps/recorder/src/config/mod.rs
View File

@@ -1,12 +1,12 @@
use figment::{
providers::{Format, Json, Yaml},
Figment,
providers::{Format, Json, Yaml},
};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use crate::{
auth::AppAuthConfig, dal::config::AppDalConfig, extract::mikan::AppMikanConfig,
graphql::config::AppGraphQLConfig,
auth::AppAuthConfig, dal::config::AppDalConfig, errors::RecorderError,
extract::mikan::AppMikanConfig, graphql::config::AppGraphQLConfig,
};
const DEFAULT_APP_SETTINGS_MIXIN: &str = include_str!("./settings_mixin.yaml");
@@ -51,7 +51,7 @@ pub fn deserialize_key_path_from_app_config<T: DeserializeOwned>(
pub trait AppConfigExt {
fn get_root_conf(&self) -> &loco_rs::config::Config;
fn get_app_conf(&self) -> loco_rs::Result<AppConfig> {
fn get_app_conf(&self) -> Result<AppConfig, RecorderError> {
let settings_str = self
.get_root_conf()
.settings
@@ -61,8 +61,7 @@ pub trait AppConfigExt {
let app_config = Figment::from(Json::string(&settings_str))
.merge(Yaml::string(DEFAULT_APP_SETTINGS_MIXIN))
.extract()
.map_err(loco_rs::Error::wrap)?;
.extract()?;
Ok(app_config)
}

6
apps/recorder/src/dal/client.rs
View File

@@ -11,7 +11,7 @@ use url::Url;
use uuid::Uuid;
use super::AppDalConfig;
use crate::{app::App, config::AppConfigExt};
use crate::{app::App, config::AppConfigExt, errors::RecorderError};
// TODO: wait app-context-trait to integrate
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -85,7 +85,7 @@ impl AppDalClient {
bucket: Option<&str>,
filename: &str,
data: Bytes,
) -> color_eyre::eyre::Result<DalStoredUrl> {
) -> Result<DalStoredUrl, RecorderError> {
match content_category {
DalContentCategory::Image => {
let fullname = [
@@ -122,7 +122,7 @@ impl AppDalClient {
subscriber_id: i32,
bucket: Option<&str>,
filename: &str,
) -> color_eyre::eyre::Result<Option<DalStoredUrl>> {
) -> Result<Option<DalStoredUrl>, RecorderError> {
match content_category {
DalContentCategory::Image => {
let fullname = [

38
apps/recorder/src/extract/errors.rs → apps/recorder/src/errors/mod.rs
View File

@@ -1,13 +1,29 @@
use std::{borrow::Cow, error::Error as StdError};
use thiserror::Error;
use thiserror::Error as ThisError;
#[derive(Error, Debug)]
pub enum ExtractError {
#[error("Extract bangumi season error: {0}")]
BangumiSeasonError(#[from] std::num::ParseIntError),
#[error("Extract file url error: {0}")]
FileUrlError(#[from] url::ParseError),
use crate::fetch::HttpClientError;
#[derive(ThisError, Debug)]
pub enum RecorderError {
#[error(transparent)]
CookieParseError(#[from] cookie::ParseError),
#[error(transparent)]
FigmentError(#[from] figment::Error),
#[error(transparent)]
SerdeJsonError(#[from] serde_json::Error),
#[error(transparent)]
ReqwestMiddlewareError(#[from] reqwest_middleware::Error),
#[error(transparent)]
ReqwestError(#[from] reqwest::Error),
#[error(transparent)]
ParseUrlError(#[from] url::ParseError),
#[error(transparent)]
OpenDALError(#[from] opendal::Error),
#[error(transparent)]
InvalidHeaderValueError(#[from] http::header::InvalidHeaderValue),
#[error(transparent)]
HttpClientError(#[from] HttpClientError),
#[error("Extract {desc} with mime error, expected {expected}, but got {found}")]
MimeError {
desc: String,
@@ -30,7 +46,7 @@ pub enum ExtractError {
},
}
impl ExtractError {
impl RecorderError {
pub fn from_mikan_meta_missing_field(field: Cow<'static, str>) -> Self {
Self::MikanMetaMissingFieldError {
field,
@@ -55,3 +71,9 @@ impl ExtractError {
}
}
}
impl From<RecorderError> for loco_rs::Error {
fn from(error: RecorderError) -> Self {
Self::wrap(error)
}
}

35
apps/recorder/src/extract/mikan/client.rs
View File

@@ -4,16 +4,30 @@ use async_trait::async_trait;
use loco_rs::app::{AppContext, Initializer};
use once_cell::sync::OnceCell;
use reqwest_middleware::ClientWithMiddleware;
use secrecy::{ExposeSecret, SecretString};
use url::Url;
use super::AppMikanConfig;
use crate::{
config::AppConfigExt,
fetch::{HttpClient, HttpClientTrait},
errors::RecorderError,
fetch::{HttpClient, HttpClientTrait, client::HttpClientCookiesAuth},
};
static APP_MIKAN_CLIENT: OnceCell<AppMikanClient> = OnceCell::new();
#[derive(Debug, Default, Clone)]
pub struct MikanAuthSecrecy {
pub cookie: SecretString,
pub user_agent: Option<String>,
}
impl MikanAuthSecrecy {
pub fn into_cookie_auth(self, url: &Url) -> Result<HttpClientCookiesAuth, RecorderError> {
HttpClientCookiesAuth::from_cookies(self.cookie.expose_secret(), url, self.user_agent)
}
}
#[derive(Debug)]
pub struct AppMikanClient {
http_client: HttpClient,
@@ -21,9 +35,8 @@ pub struct AppMikanClient {
}
impl AppMikanClient {
pub fn new(config: AppMikanConfig) -> loco_rs::Result<Self> {
let http_client =
HttpClient::from_config(config.http_client).map_err(loco_rs::Error::wrap)?;
pub fn new(config: AppMikanConfig) -> Result<Self, RecorderError> {
let http_client = HttpClient::from_config(config.http_client)?;
let base_url = config.base_url;
Ok(Self {
http_client,
@@ -31,6 +44,16 @@ impl AppMikanClient {
})
}
pub fn fork_with_auth(&self, secrecy: MikanAuthSecrecy) -> Result<Self, RecorderError> {
let cookie_auth = secrecy.into_cookie_auth(&self.base_url)?;
let fork = self.http_client.fork().attach_secrecy(cookie_auth);
Ok(Self {
http_client: HttpClient::from_fork(fork)?,
base_url: self.base_url.clone(),
})
}
pub fn app_instance() -> &'static AppMikanClient {
APP_MIKAN_CLIENT
.get()
@@ -40,6 +63,10 @@ impl AppMikanClient {
pub fn base_url(&self) -> &Url {
&self.base_url
}
pub fn client(&self) -> &HttpClient {
&self.http_client
}
}
impl Deref for AppMikanClient {

2
apps/recorder/src/extract/mikan/mod.rs
View File

@@ -4,7 +4,7 @@ pub mod constants;
pub mod rss_extract;
pub mod web_extract;
pub use client::{AppMikanClient, AppMikanClientInitializer};
pub use client::{AppMikanClient, AppMikanClientInitializer, MikanAuthSecrecy};
pub use config::AppMikanConfig;
pub use constants::MIKAN_BUCKET_KEY;
pub use rss_extract::{

28
apps/recorder/src/extract/mikan/rss_extract.rs
View File

@@ -9,12 +9,10 @@ use tracing::instrument;
use url::Url;
use crate::{
extract::{
errors::ExtractError,
mikan::{
AppMikanClient,
web_extract::{MikanEpisodeHomepage, parse_mikan_episode_id_from_homepage},
},
errors::RecorderError,
extract::mikan::{
AppMikanClient,
web_extract::{MikanEpisodeHomepage, extract_mikan_episode_id_from_homepage},
},
fetch::bytes::fetch_bytes,
sync::core::BITTORRENT_MIME_TYPE,
@@ -103,16 +101,16 @@ impl MikanRssChannel {
}
impl TryFrom<rss::Item> for MikanRssItem {
type Error = ExtractError;
type Error = RecorderError;
fn try_from(item: rss::Item) -> Result<Self, Self::Error> {
let enclosure = item.enclosure.ok_or_else(|| {
ExtractError::from_mikan_rss_invalid_field(Cow::Borrowed("enclosure"))
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("enclosure"))
})?;
let mime_type = enclosure.mime_type;
if mime_type != BITTORRENT_MIME_TYPE {
return Err(ExtractError::MimeError {
return Err(RecorderError::MimeError {
expected: String::from(BITTORRENT_MIME_TYPE),
found: mime_type.to_string(),
desc: String::from("MikanRssItem"),
@@ -120,11 +118,11 @@ impl TryFrom<rss::Item> for MikanRssItem {
}
let title = item.title.ok_or_else(|| {
ExtractError::from_mikan_rss_invalid_field(Cow::Borrowed("title:title"))
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("title:title"))
})?;
let enclosure_url = Url::parse(&enclosure.url).map_err(|inner| {
ExtractError::from_mikan_rss_invalid_field_and_source(
RecorderError::from_mikan_rss_invalid_field_and_source(
Cow::Borrowed("enclosure_url:enclosure.link"),
Box::new(inner),
)
@@ -134,13 +132,13 @@ impl TryFrom<rss::Item> for MikanRssItem {
.link
.and_then(|link| Url::parse(&link).ok())
.ok_or_else(|| {
ExtractError::from_mikan_rss_invalid_field(Cow::Borrowed("homepage:link"))
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("homepage:link"))
})?;
let MikanEpisodeHomepage {
mikan_episode_id, ..
} = parse_mikan_episode_id_from_homepage(&homepage).ok_or_else(|| {
ExtractError::from_mikan_rss_invalid_field(Cow::Borrowed("mikan_episode_id"))
} = extract_mikan_episode_id_from_homepage(&homepage).ok_or_else(|| {
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("mikan_episode_id"))
})?;
Ok(MikanRssItem {
@@ -329,7 +327,7 @@ pub async fn extract_mikan_rss_channel_from_rss_link(
},
))
} else {
Err(ExtractError::MikanRssInvalidFormatError)
Err(RecorderError::MikanRssInvalidFormatError)
.inspect_err(|error| {
tracing::warn!(error = %error);
})

328
apps/recorder/src/extract/mikan/web_extract.rs
View File

@@ -1,9 +1,8 @@
use std::borrow::Cow;
use bytes::Bytes;
use color_eyre::eyre;
use itertools::Itertools;
use loco_rs::app::AppContext;
use reqwest::IntoUrl;
use scraper::{Html, Selector};
use tracing::instrument;
use url::Url;
@@ -14,8 +13,8 @@ use super::{
use crate::{
app::AppContextExt,
dal::DalContentCategory,
errors::RecorderError,
extract::{
errors::ExtractError,
html::{extract_background_image_src_from_style_attr, extract_inner_text_from_element_ref},
media::extract_image_src_from_str,
},
@@ -63,38 +62,32 @@ pub struct MikanBangumiHomepage {
}
pub fn build_mikan_bangumi_homepage(
mikan_base_url: impl IntoUrl,
mikan_base_url: Url,
mikan_bangumi_id: &str,
mikan_fansub_id: Option<&str>,
) -> eyre::Result<Url> {
let mut url = mikan_base_url.into_url()?;
) -> Url {
let mut url = mikan_base_url;
url.set_path(&format!("/Home/Bangumi/{mikan_bangumi_id}"));
url.set_fragment(mikan_fansub_id);
Ok(url)
url
}
pub fn build_mikan_episode_homepage(
mikan_base_url: impl IntoUrl,
mikan_episode_id: &str,
) -> eyre::Result<Url> {
let mut url = mikan_base_url.into_url()?;
pub fn build_mikan_episode_homepage(mikan_base_url: Url, mikan_episode_id: &str) -> Url {
let mut url = mikan_base_url;
url.set_path(&format!("/Home/Episode/{mikan_episode_id}"));
Ok(url)
url
}
pub fn build_mikan_bangumi_expand_info_url(
mikan_base_url: impl IntoUrl,
mikan_bangumi_id: &str,
) -> eyre::Result<Url> {
let mut url = mikan_base_url.into_url()?;
pub fn build_mikan_bangumi_expand_info_url(mikan_base_url: Url, mikan_bangumi_id: &str) -> Url {
let mut url = mikan_base_url;
url.set_path("/ExpandBangumi");
url.query_pairs_mut()
.append_pair("bangumiId", mikan_bangumi_id)
.append_pair("showSubscribed", "true");
Ok(url)
url
}
pub fn parse_mikan_bangumi_id_from_homepage(url: &Url) -> Option<MikanBangumiHomepage> {
pub fn extract_mikan_bangumi_id_from_homepage(url: &Url) -> Option<MikanBangumiHomepage> {
if url.path().starts_with("/Home/Bangumi/") {
let mikan_bangumi_id = url.path().replace("/Home/Bangumi/", "");
@@ -107,7 +100,7 @@ pub fn parse_mikan_bangumi_id_from_homepage(url: &Url) -> Option<MikanBangumiHom
}
}
pub fn parse_mikan_episode_id_from_homepage(url: &Url) -> Option<MikanEpisodeHomepage> {
pub fn extract_mikan_episode_id_from_homepage(url: &Url) -> Option<MikanEpisodeHomepage> {
if url.path().starts_with("/Home/Episode/") {
let mikan_episode_id = url.path().replace("/Home/Episode/", "");
Some(MikanEpisodeHomepage { mikan_episode_id })
@@ -119,7 +112,7 @@ pub fn parse_mikan_episode_id_from_homepage(url: &Url) -> Option<MikanEpisodeHom
pub async fn extract_mikan_poster_meta_from_src(
http_client: &AppMikanClient,
origin_poster_src_url: Url,
) -> eyre::Result<MikanBangumiPosterMeta> {
) -> Result<MikanBangumiPosterMeta, RecorderError> {
let poster_data = fetch_image(http_client, origin_poster_src_url.clone()).await?;
Ok(MikanBangumiPosterMeta {
origin_poster_src: origin_poster_src_url,
@@ -132,7 +125,7 @@ pub async fn extract_mikan_bangumi_poster_meta_from_src_with_cache(
ctx: &AppContext,
origin_poster_src_url: Url,
subscriber_id: i32,
) -> eyre::Result<MikanBangumiPosterMeta> {
) -> Result<MikanBangumiPosterMeta, RecorderError> {
let dal_client = ctx.get_dal_client();
let mikan_client = ctx.get_mikan_client();
if let Some(poster_src) = dal_client
@@ -174,7 +167,7 @@ pub async fn extract_mikan_bangumi_poster_meta_from_src_with_cache(
pub async fn extract_mikan_episode_meta_from_episode_homepage(
http_client: &AppMikanClient,
mikan_episode_homepage_url: Url,
) -> eyre::Result<MikanEpisodeMeta> {
) -> Result<MikanEpisodeMeta, RecorderError> {
let mikan_base_url = Url::parse(&mikan_episode_homepage_url.origin().unicode_serialization())?;
let content = fetch_html(http_client, mikan_episode_homepage_url.as_str()).await?;
@@ -190,7 +183,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
.select(bangumi_title_selector)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -206,13 +199,13 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
.and_then(|s| mikan_episode_homepage_url.join(s).ok())
.and_then(|rss_link_url| extract_mikan_bangumi_id_from_rss_link(&rss_link_url))
.ok_or_else(|| {
ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
let mikan_fansub_id = mikan_fansub_id
.ok_or_else(|| {
ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_fansub_id"))
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_fansub_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
@@ -220,16 +213,16 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
.select(&Selector::parse("title").unwrap())
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("episode_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("episode_title")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
let MikanEpisodeHomepage {
mikan_episode_id, ..
} = parse_mikan_episode_id_from_homepage(&mikan_episode_homepage_url)
} = extract_mikan_episode_id_from_homepage(&mikan_episode_homepage_url)
.ok_or_else(|| {
ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_episode_id"))
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_episode_id"))
})
.inspect_err(|error| {
tracing::warn!(error = %error);
@@ -242,7 +235,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("fansub_name")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("fansub_name")))
.inspect_err(|error| {
tracing::warn!(error = %error);
})?;
@@ -285,7 +278,7 @@ pub async fn extract_mikan_episode_meta_from_episode_homepage(
pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
http_client: &AppMikanClient,
mikan_bangumi_homepage_url: Url,
) -> eyre::Result<MikanBangumiMeta> {
) -> Result<MikanBangumiMeta, RecorderError> {
let mikan_base_url = Url::parse(&mikan_bangumi_homepage_url.origin().unicode_serialization())?;
let content = fetch_html(http_client, mikan_bangumi_homepage_url.as_str()).await?;
let html = Html::parse_document(&content);
@@ -299,7 +292,7 @@ pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
.select(bangumi_title_selector)
.next()
.map(extract_inner_text_from_element_ref)
.ok_or_else(|| ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.ok_or_else(|| RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("bangumi_title")))
.inspect_err(|error| tracing::warn!(error = %error))?;
let mikan_bangumi_id = html
@@ -314,7 +307,7 @@ pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
}| mikan_bangumi_id,
)
.ok_or_else(|| {
ExtractError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("mikan_bangumi_id"))
})
.inspect_err(|error| tracing::error!(error = %error))?;
@@ -367,97 +360,120 @@ pub async fn extract_mikan_bangumi_meta_from_bangumi_homepage(
pub async fn extract_mikan_bangumis_meta_from_my_bangumi_page(
http_client: &AppMikanClient,
my_bangumi_page_url: Url,
) -> eyre::Result<Vec<MikanBangumiMeta>> {
) -> Result<Vec<MikanBangumiMeta>, RecorderError> {
let mikan_base_url = Url::parse(&my_bangumi_page_url.origin().unicode_serialization())?;
let content = fetch_html(http_client, my_bangumi_page_url.clone()).await?;
let bangumi_container_selector = &Selector::parse(".sk-bangumi .an-ul>li").unwrap();
let bangumi_info_selector = &Selector::parse(".an-info a.an-text").unwrap();
let bangumi_poster_selector =
&Selector::parse("span[data-src][data-bangumiid], span[data-bangumiid][style]").unwrap();
let fansub_container_selector =
&Selector::parse(".js-expand_bangumi-subgroup.js-subscribed").unwrap();
let fansub_title_selector = &Selector::parse(".tag-res-name[title]").unwrap();
let fansub_id_selector =
&Selector::parse(".active[data-subtitlegroupid][data-bangumiid]").unwrap();
let html = Html::parse_document(&content);
let bangumi_iters = {
let bangumi_container_selector = &Selector::parse(".sk-bangumi .an-ul>li").unwrap();
let bangumi_info_selector = &Selector::parse(".an-info a.an-text").unwrap();
let bangumi_poster_selector =
&Selector::parse("span[data-src][data-bangumiid], span[data-bangumiid][style]")
.unwrap();
let html = Html::parse_document(&content);
html.select(bangumi_container_selector)
.filter_map(|bangumi_elem| {
let title_and_href_elem = bangumi_elem.select(bangumi_info_selector).next();
let poster_elem = bangumi_elem.select(bangumi_poster_selector).next();
if let (Some(bangumi_home_page_url), Some(bangumi_title)) = (
title_and_href_elem.and_then(|elem| elem.attr("href")),
title_and_href_elem.and_then(|elem| elem.attr("title")),
) {
let origin_poster_src = poster_elem.and_then(|ele| {
ele.attr("data-src")
.and_then(|data_src| {
extract_image_src_from_str(data_src, &mikan_base_url)
})
.or_else(|| {
ele.attr("style").and_then(|style| {
extract_background_image_src_from_style_attr(
style,
&mikan_base_url,
)
})
})
});
let bangumi_title = bangumi_title.to_string();
let bangumi_home_page_url =
my_bangumi_page_url.join(bangumi_home_page_url).ok()?;
let MikanBangumiHomepage {
mikan_bangumi_id, ..
} = extract_mikan_bangumi_id_from_homepage(&bangumi_home_page_url)?;
if let Some(origin_poster_src) = origin_poster_src.as_ref() {
tracing::trace!(
origin_poster_src = origin_poster_src.as_str(),
bangumi_title,
mikan_bangumi_id,
"bangumi info extracted"
);
} else {
tracing::warn!(
bangumi_title,
mikan_bangumi_id,
"bangumi info extracted, but failed to extract poster_src"
);
}
let bangumi_expand_info_url = build_mikan_bangumi_expand_info_url(
mikan_base_url.clone(),
&mikan_bangumi_id,
);
Some((
bangumi_title,
mikan_bangumi_id,
bangumi_expand_info_url,
origin_poster_src,
))
} else {
None
}
})
.collect_vec()
};
let mut bangumi_list = vec![];
for bangumi_elem in html.select(bangumi_container_selector) {
let title_and_href_elem = bangumi_elem.select(bangumi_info_selector).next();
let poster_elem = bangumi_elem.select(bangumi_poster_selector).next();
if let (Some(bangumi_home_page_url), Some(bangumi_title)) = (
title_and_href_elem.and_then(|elem| elem.attr("href")),
title_and_href_elem.and_then(|elem| elem.attr("title")),
) {
let origin_poster_src = poster_elem.and_then(|ele| {
ele.attr("data-src")
.and_then(|data_src| extract_image_src_from_str(data_src, &mikan_base_url))
.or_else(|| {
ele.attr("style").and_then(|style| {
extract_background_image_src_from_style_attr(style, &mikan_base_url)
})
})
});
let bangumi_home_page_url = my_bangumi_page_url.join(bangumi_home_page_url)?;
if let Some(MikanBangumiHomepage {
ref mikan_bangumi_id,
..
}) = parse_mikan_bangumi_id_from_homepage(&bangumi_home_page_url)
{
if let Some(origin_poster_src) = origin_poster_src.as_ref() {
tracing::trace!(
origin_poster_src = origin_poster_src.as_str(),
bangumi_title,
mikan_bangumi_id,
"bangumi info extracted"
);
} else {
tracing::warn!(
bangumi_title,
mikan_bangumi_id,
"bangumi info extracted, but failed to extract poster_src"
);
}
let bangumi_expand_info_url =
build_mikan_bangumi_expand_info_url(mikan_base_url.clone(), mikan_bangumi_id)?;
let bangumi_expand_info_content =
fetch_html(http_client, bangumi_expand_info_url).await?;
let bangumi_expand_info_fragment =
Html::parse_fragment(&bangumi_expand_info_content);
for fansub_info in bangumi_expand_info_fragment.select(fansub_container_selector) {
if let (Some(fansub_name), Some(mikan_fansub_id)) = (
fansub_info
.select(fansub_title_selector)
.next()
.and_then(|ele| ele.attr("title")),
fansub_info
.select(fansub_id_selector)
.next()
.and_then(|ele| ele.attr("data-subtitlegroupid")),
) {
tracing::trace!(
fansub_name = &fansub_name,
mikan_fansub_id,
"subscribed fansub extracted"
);
bangumi_list.push(MikanBangumiMeta {
homepage: build_mikan_bangumi_homepage(
mikan_base_url.clone(),
mikan_bangumi_id.as_str(),
Some(mikan_fansub_id),
)?,
bangumi_title: bangumi_title.to_string(),
mikan_bangumi_id: mikan_bangumi_id.to_string(),
mikan_fansub_id: Some(mikan_fansub_id.to_string()),
fansub: Some(fansub_name.to_string()),
origin_poster_src: origin_poster_src.clone(),
})
}
}
for (bangumi_title, mikan_bangumi_id, bangumi_expand_info_url, origin_poster_src) in
bangumi_iters
{
let bangumi_expand_info_content = fetch_html(http_client, bangumi_expand_info_url).await?;
let bangumi_expand_info_fragment = Html::parse_fragment(&bangumi_expand_info_content);
for fansub_info in bangumi_expand_info_fragment.select(fansub_container_selector) {
if let (Some(fansub_name), Some(mikan_fansub_id)) = (
fansub_info
.select(fansub_title_selector)
.next()
.and_then(|ele| ele.attr("title")),
fansub_info
.select(fansub_id_selector)
.next()
.and_then(|ele| ele.attr("data-subtitlegroupid")),
) {
tracing::trace!(
fansub_name = &fansub_name,
mikan_fansub_id,
"subscribed fansub extracted"
);
bangumi_list.push(MikanBangumiMeta {
homepage: build_mikan_bangumi_homepage(
mikan_base_url.clone(),
mikan_bangumi_id.as_str(),
Some(mikan_fansub_id),
),
bangumi_title: bangumi_title.to_string(),
mikan_bangumi_id: mikan_bangumi_id.to_string(),
mikan_fansub_id: Some(mikan_fansub_id.to_string()),
fansub: Some(fansub_name.to_string()),
origin_poster_src: origin_poster_src.clone(),
})
}
}
}
@@ -469,14 +485,18 @@ pub async fn extract_mikan_bangumis_meta_from_my_bangumi_page(
mod test {
#![allow(unused_variables)]
use color_eyre::eyre;
use http::header;
use rstest::{fixture, rstest};
use secrecy::SecretString;
use tracing::Level;
use url::Url;
use zune_image::{codecs::ImageFormat, image::Image};
use super::*;
use crate::{
extract::mikan::web_extract::extract_mikan_bangumis_meta_from_my_bangumi_page,
extract::mikan::{
MikanAuthSecrecy, web_extract::extract_mikan_bangumis_meta_from_my_bangumi_page,
},
test_utils::{mikan::build_testing_mikan_client, tracing::init_testing_tracing},
};
@@ -604,33 +624,75 @@ mod test {
let mikan_base_url = Url::parse(&mikan_server.url())?;
let mikan_client = build_testing_mikan_client(mikan_base_url.clone())?;
let my_bangumi_page_url = mikan_base_url.join("/Home/MyBangumi")?;
let my_bangumi_mock = mikan_server
.mock("GET", my_bangumi_page_url.path())
.with_body_from_file("tests/resources/mikan/MyBangumi.htm")
.create_async()
.await;
let mikan_client = build_testing_mikan_client(mikan_base_url.clone())?;
let expand_bangumi_mock = mikan_server
.mock("GET", "/ExpandBangumi")
.match_query(mockito::Matcher::Any)
.with_body_from_file("tests/resources/mikan/ExpandBangumi.htm")
.create_async()
.await;
{
let my_bangumi_without_cookie_mock = mikan_server
.mock("GET", my_bangumi_page_url.path())
.match_header(header::COOKIE, mockito::Matcher::Missing)
.with_body_from_file("tests/resources/mikan/MyBangumi-noauth.htm")
.create_async()
.await;
let bangumi_metas =
extract_mikan_bangumis_meta_from_my_bangumi_page(&mikan_client, my_bangumi_page_url)
.await?;
let bangumi_metas = extract_mikan_bangumis_meta_from_my_bangumi_page(
&mikan_client,
my_bangumi_page_url.clone(),
)
.await?;
assert!(!bangumi_metas.is_empty());
assert!(bangumi_metas.is_empty());
assert!(bangumi_metas[0].origin_poster_src.is_some());
assert!(my_bangumi_without_cookie_mock.matched_async().await);
}
{
let my_bangumi_with_cookie_mock = mikan_server
.mock("GET", my_bangumi_page_url.path())
.match_header(
header::COOKIE,
mockito::Matcher::AllOf(vec![
mockito::Matcher::Regex(String::from(".*\\.AspNetCore\\.Antiforgery.*")),
mockito::Matcher::Regex(String::from(
".*\\.AspNetCore\\.Identity\\.Application.*",
)),
]),
)
.with_body_from_file("tests/resources/mikan/MyBangumi.htm")
.create_async()
.await;
my_bangumi_mock.expect(1);
expand_bangumi_mock.expect(bangumi_metas.len());
let expand_bangumi_mock = mikan_server
.mock("GET", "/ExpandBangumi")
.match_query(mockito::Matcher::Any)
.with_body_from_file("tests/resources/mikan/ExpandBangumi.htm")
.create_async()
.await;
let mikan_client_with_cookie = mikan_client.fork_with_auth(MikanAuthSecrecy {
cookie: SecretString::from(
"mikan-announcement=1; .AspNetCore.Antiforgery.abc=abc; \
.AspNetCore.Identity.Application=abc; ",
),
user_agent: Some(String::from(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like \
Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/133.0.0.0",
)),
})?;
let bangumi_metas = extract_mikan_bangumis_meta_from_my_bangumi_page(
&mikan_client_with_cookie,
my_bangumi_page_url,
)
.await?;
assert!(!bangumi_metas.is_empty());
assert!(bangumi_metas[0].origin_poster_src.is_some());
assert!(my_bangumi_with_cookie_mock.matched_async().await);
expand_bangumi_mock.expect(bangumi_metas.len());
}
Ok(())
}

1
apps/recorder/src/extract/mod.rs
View File

@@ -1,5 +1,4 @@
pub mod defs;
pub mod errors;
pub mod html;
pub mod http;
pub mod media;

3
apps/recorder/src/fetch/bytes.rs
View File

@@ -2,11 +2,12 @@ use bytes::Bytes;
use reqwest::IntoUrl;
use super::client::HttpClientTrait;
use crate::errors::RecorderError;
pub async fn fetch_bytes<T: IntoUrl, H: HttpClientTrait>(
client: &H,
url: T,
) -> color_eyre::eyre::Result<Bytes> {
) -> Result<Bytes, RecorderError> {
let bytes = client
.get(url)
.send()

244
apps/recorder/src/fetch/client.rs
View File

@@ -1,244 +0,0 @@
use std::{fmt::Debug, ops::Deref, sync::Arc, time::Duration};
use async_trait::async_trait;
use axum::http::{self, Extensions};
use http_cache_reqwest::{
CACacheManager, Cache, CacheManager, CacheMode, HttpCache, HttpCacheOptions, MokaManager,
};
use leaky_bucket::RateLimiter;
use reqwest::{ClientBuilder, Request, Response};
use reqwest_middleware::{
ClientBuilder as ClientWithMiddlewareBuilder, ClientWithMiddleware, Next,
};
use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
use reqwest_tracing::TracingMiddleware;
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use thiserror::Error;
use super::get_random_mobile_ua;
use crate::app::App;
pub struct RateLimiterMiddleware {
rate_limiter: RateLimiter,
}
#[async_trait]
impl reqwest_middleware::Middleware for RateLimiterMiddleware {
async fn handle(
&self,
req: Request,
extensions: &'_ mut Extensions,
next: Next<'_>,
) -> reqwest_middleware::Result<Response> {
self.rate_limiter.acquire_one().await;
next.run(req, extensions).await
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum HttpClientCacheBackendConfig {
Moka { cache_size: u64 },
CACache { cache_path: String },
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum HttpClientCachePresetConfig {
#[serde(rename = "rfc7234")]
RFC7234,
}
impl Default for HttpClientCachePresetConfig {
fn default() -> Self {
Self::RFC7234
}
}
#[serde_as]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct HttpClientConfig {
pub exponential_backoff_max_retries: Option<u32>,
pub leaky_bucket_max_tokens: Option<u32>,
pub leaky_bucket_initial_tokens: Option<u32>,
pub leaky_bucket_refill_tokens: Option<u32>,
#[serde_as(as = "Option<serde_with::DurationMilliSeconds>")]
pub leaky_bucket_refill_interval: Option<Duration>,
pub user_agent: Option<String>,
pub cache_backend: Option<HttpClientCacheBackendConfig>,
pub cache_preset: Option<HttpClientCachePresetConfig>,
}
struct CacheBackend(Box<dyn CacheManager>);
impl CacheBackend {
fn new<T: CacheManager>(backend: T) -> Self {
Self(Box::new(backend))
}
}
#[async_trait::async_trait]
impl CacheManager for CacheBackend {
async fn get(
&self,
cache_key: &str,
) -> http_cache::Result<Option<(http_cache::HttpResponse, http_cache_semantics::CachePolicy)>>
{
self.0.get(cache_key).await
}
/// Attempts to cache a response and related policy.
async fn put(
&self,
cache_key: String,
res: http_cache::HttpResponse,
policy: http_cache_semantics::CachePolicy,
) -> http_cache::Result<http_cache::HttpResponse> {
self.0.put(cache_key, res, policy).await
}
/// Attempts to remove a record from cache.
async fn delete(&self, cache_key: &str) -> http_cache::Result<()> {
self.0.delete(cache_key).await
}
}
#[derive(Debug, Error)]
pub enum HttpClientError {
#[error(transparent)]
ReqwestError(#[from] reqwest::Error),
#[error(transparent)]
ReqwestMiddlewareError(#[from] reqwest_middleware::Error),
#[error(transparent)]
HttpError(#[from] http::Error),
}
pub trait HttpClientTrait: Deref<Target = ClientWithMiddleware> {}
pub struct HttpClient {
client: ClientWithMiddleware,
pub config: HttpClientConfig,
}
impl Debug for HttpClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpClient")
.field("config", &self.config)
.finish()
}
}
impl From<HttpClient> for ClientWithMiddleware {
fn from(val: HttpClient) -> Self {
val.client
}
}
impl Deref for HttpClient {
type Target = ClientWithMiddleware;
fn deref(&self) -> &Self::Target {
&self.client
}
}
impl HttpClient {
pub fn from_config(config: HttpClientConfig) -> Result<Self, HttpClientError> {
let reqwest_client_builder = ClientBuilder::new().user_agent(
config
.user_agent
.as_deref()
.unwrap_or_else(|| get_random_mobile_ua()),
);
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client_builder =
reqwest_client_builder.redirect(reqwest::redirect::Policy::none());
let reqwest_client = reqwest_client_builder.build()?;
let mut reqwest_with_middleware_builder =
ClientWithMiddlewareBuilder::new(reqwest_client).with(TracingMiddleware::default());
if let Some(ref x) = config.exponential_backoff_max_retries {
let retry_policy = ExponentialBackoff::builder().build_with_max_retries(*x);
reqwest_with_middleware_builder = reqwest_with_middleware_builder
.with(RetryTransientMiddleware::new_with_policy(retry_policy));
}
if let (None, None, None, None) = (
config.leaky_bucket_initial_tokens.as_ref(),
config.leaky_bucket_refill_tokens.as_ref(),
config.leaky_bucket_refill_interval.as_ref(),
config.leaky_bucket_max_tokens.as_ref(),
) {
} else {
let mut rate_limiter_builder = RateLimiter::builder();
if let Some(ref x) = config.leaky_bucket_max_tokens {
rate_limiter_builder.max(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_initial_tokens {
rate_limiter_builder.initial(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_refill_tokens {
rate_limiter_builder.refill(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_refill_interval {
rate_limiter_builder.interval(*x);
}
let rate_limiter = rate_limiter_builder.build();
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with(RateLimiterMiddleware { rate_limiter });
}
if let (None, None) = (config.cache_backend.as_ref(), config.cache_preset.as_ref()) {
} else {
let cache_preset = config.cache_preset.as_ref().cloned().unwrap_or_default();
let cache_backend = config
.cache_backend
.as_ref()
.map(|b| match b {
HttpClientCacheBackendConfig::CACache { cache_path } => {
let path = std::path::PathBuf::from(
App::get_working_root().join(cache_path).as_str(),
);
CacheBackend::new(CACacheManager { path })
}
HttpClientCacheBackendConfig::Moka { cache_size } => {
CacheBackend::new(MokaManager {
cache: Arc::new(moka::future::Cache::new(u64::max(*cache_size, 1))),
})
}
})
.unwrap_or_else(|| CacheBackend::new(MokaManager::default()));
let http_cache = match cache_preset {
HttpClientCachePresetConfig::RFC7234 => HttpCache {
mode: CacheMode::Default,
manager: cache_backend,
options: HttpCacheOptions::default(),
},
};
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with(Cache(http_cache));
}
let reqwest_with_middleware = reqwest_with_middleware_builder.build();
Ok(Self {
client: reqwest_with_middleware,
config,
})
}
}
impl Default for HttpClient {
fn default() -> Self {
HttpClient::from_config(Default::default()).expect("Failed to create default HttpClient")
}
}
impl HttpClientTrait for HttpClient {}

329
apps/recorder/src/fetch/client/core.rs Normal file
View File

@@ -0,0 +1,329 @@
use std::{fmt::Debug, ops::Deref, sync::Arc, time::Duration};
use async_trait::async_trait;
use axum::http::{self, Extensions};
use http_cache_reqwest::{
CACacheManager, Cache, CacheManager, CacheMode, HttpCache, HttpCacheOptions, MokaManager,
};
use leaky_bucket::RateLimiter;
use reqwest::{ClientBuilder, Request, Response};
use reqwest_middleware::{
ClientBuilder as ClientWithMiddlewareBuilder, ClientWithMiddleware, Middleware, Next,
};
use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
use reqwest_tracing::TracingMiddleware;
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use thiserror::Error;
use super::HttpClientSecrecyDataTrait;
use crate::{app::App, fetch::get_random_mobile_ua};
pub struct RateLimiterMiddleware {
rate_limiter: RateLimiter,
}
#[async_trait]
impl Middleware for RateLimiterMiddleware {
async fn handle(
&self,
req: Request,
extensions: &'_ mut Extensions,
next: Next<'_>,
) -> reqwest_middleware::Result<Response> {
self.rate_limiter.acquire_one().await;
next.run(req, extensions).await
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum HttpClientCacheBackendConfig {
Moka { cache_size: u64 },
CACache { cache_path: String },
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum HttpClientCachePresetConfig {
#[serde(rename = "rfc7234")]
RFC7234,
}
impl Default for HttpClientCachePresetConfig {
fn default() -> Self {
Self::RFC7234
}
}
#[serde_as]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct HttpClientConfig {
pub exponential_backoff_max_retries: Option<u32>,
pub leaky_bucket_max_tokens: Option<u32>,
pub leaky_bucket_initial_tokens: Option<u32>,
pub leaky_bucket_refill_tokens: Option<u32>,
#[serde_as(as = "Option<serde_with::DurationMilliSeconds>")]
pub leaky_bucket_refill_interval: Option<Duration>,
pub user_agent: Option<String>,
pub cache_backend: Option<HttpClientCacheBackendConfig>,
pub cache_preset: Option<HttpClientCachePresetConfig>,
}
pub(crate) struct CacheBackend(Box<dyn CacheManager>);
impl CacheBackend {
pub(crate) fn new<T: CacheManager>(backend: T) -> Self {
Self(Box::new(backend))
}
}
#[async_trait::async_trait]
impl CacheManager for CacheBackend {
async fn get(
&self,
cache_key: &str,
) -> http_cache::Result<Option<(http_cache::HttpResponse, http_cache_semantics::CachePolicy)>>
{
self.0.get(cache_key).await
}
/// Attempts to cache a response and related policy.
async fn put(
&self,
cache_key: String,
res: http_cache::HttpResponse,
policy: http_cache_semantics::CachePolicy,
) -> http_cache::Result<http_cache::HttpResponse> {
self.0.put(cache_key, res, policy).await
}
/// Attempts to remove a record from cache.
async fn delete(&self, cache_key: &str) -> http_cache::Result<()> {
self.0.delete(cache_key).await
}
}
#[derive(Debug, Error)]
pub enum HttpClientError {
#[error(transparent)]
ReqwestError(#[from] reqwest::Error),
#[error(transparent)]
ReqwestMiddlewareError(#[from] reqwest_middleware::Error),
#[error(transparent)]
HttpError(#[from] http::Error),
}
pub trait HttpClientTrait: Deref<Target = ClientWithMiddleware> + Debug {}
pub struct HttpClientFork {
pub client_builder: ClientBuilder,
pub middleware_stack: Vec<Arc<dyn Middleware>>,
pub config: HttpClientConfig,
}
impl HttpClientFork {
pub fn attach_secrecy<S: HttpClientSecrecyDataTrait>(self, secrecy: S) -> Self {
let mut fork = self;
fork.client_builder = secrecy.attach_secrecy_to_client(fork.client_builder);
fork
}
}
pub struct HttpClient {
client: ClientWithMiddleware,
middleware_stack: Vec<Arc<dyn Middleware>>,
pub config: HttpClientConfig,
}
impl Debug for HttpClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpClient")
.field("config", &self.config)
.finish()
}
}
impl From<HttpClient> for ClientWithMiddleware {
fn from(val: HttpClient) -> Self {
val.client
}
}
impl Deref for HttpClient {
type Target = ClientWithMiddleware;
fn deref(&self) -> &Self::Target {
&self.client
}
}
impl HttpClient {
pub fn from_config(config: HttpClientConfig) -> Result<Self, HttpClientError> {
let mut middleware_stack: Vec<Arc<dyn Middleware>> = vec![];
let reqwest_client_builder = ClientBuilder::new().user_agent(
config
.user_agent
.as_deref()
.unwrap_or_else(|| get_random_mobile_ua()),
);
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client_builder =
reqwest_client_builder.redirect(reqwest::redirect::Policy::none());
let reqwest_client = reqwest_client_builder.build()?;
let mut reqwest_with_middleware_builder = ClientWithMiddlewareBuilder::new(reqwest_client);
{
let tracing_middleware = Arc::new(TracingMiddleware::default());
middleware_stack.push(tracing_middleware.clone());
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with_arc(tracing_middleware)
}
{
if let Some(ref x) = config.exponential_backoff_max_retries {
let retry_policy = ExponentialBackoff::builder().build_with_max_retries(*x);
let retry_transient_middleware =
Arc::new(RetryTransientMiddleware::new_with_policy(retry_policy));
middleware_stack.push(retry_transient_middleware.clone());
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with_arc(retry_transient_middleware);
}
}
{
if let (None, None, None, None) = (
config.leaky_bucket_initial_tokens.as_ref(),
config.leaky_bucket_refill_tokens.as_ref(),
config.leaky_bucket_refill_interval.as_ref(),
config.leaky_bucket_max_tokens.as_ref(),
) {
} else {
let mut rate_limiter_builder = RateLimiter::builder();
if let Some(ref x) = config.leaky_bucket_max_tokens {
rate_limiter_builder.max(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_initial_tokens {
rate_limiter_builder.initial(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_refill_tokens {
rate_limiter_builder.refill(*x as usize);
}
if let Some(ref x) = config.leaky_bucket_refill_interval {
rate_limiter_builder.interval(*x);
}
let rate_limiter = rate_limiter_builder.build();
let rate_limiter_middleware = Arc::new(RateLimiterMiddleware { rate_limiter });
middleware_stack.push(rate_limiter_middleware.clone());
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with_arc(rate_limiter_middleware);
}
}
{
if let (None, None) = (config.cache_backend.as_ref(), config.cache_preset.as_ref()) {
} else {
let cache_preset = config.cache_preset.as_ref().cloned().unwrap_or_default();
let cache_backend = config
.cache_backend
.as_ref()
.map(|b| match b {
HttpClientCacheBackendConfig::CACache { cache_path } => {
let path = std::path::PathBuf::from(
App::get_working_root().join(cache_path).as_str(),
);
CacheBackend::new(CACacheManager { path })
}
HttpClientCacheBackendConfig::Moka { cache_size } => {
CacheBackend::new(MokaManager {
cache: Arc::new(moka::future::Cache::new(*cache_size)),
})
}
})
.unwrap_or_else(|| CacheBackend::new(MokaManager::default()));
let http_cache = match cache_preset {
HttpClientCachePresetConfig::RFC7234 => HttpCache {
mode: CacheMode::Default,
manager: cache_backend,
options: HttpCacheOptions::default(),
},
};
let http_cache_middleware = Arc::new(Cache(http_cache));
middleware_stack.push(http_cache_middleware.clone());
reqwest_with_middleware_builder =
reqwest_with_middleware_builder.with_arc(http_cache_middleware);
}
}
let reqwest_with_middleware = reqwest_with_middleware_builder.build();
Ok(Self {
client: reqwest_with_middleware,
middleware_stack,
config,
})
}
pub fn fork(&self) -> HttpClientFork {
let reqwest_client_builder = ClientBuilder::new().user_agent(
self.config
.user_agent
.as_deref()
.unwrap_or_else(|| get_random_mobile_ua()),
);
#[cfg(not(target_arch = "wasm32"))]
let reqwest_client_builder =
reqwest_client_builder.redirect(reqwest::redirect::Policy::none());
HttpClientFork {
client_builder: reqwest_client_builder,
middleware_stack: self.middleware_stack.clone(),
config: self.config.clone(),
}
}
pub fn from_fork(fork: HttpClientFork) -> Result<Self, HttpClientError> {
let HttpClientFork {
client_builder,
middleware_stack,
config,
} = fork;
let reqwest_client = client_builder.build()?;
let mut reqwest_with_middleware_builder = ClientWithMiddlewareBuilder::new(reqwest_client);
for m in &middleware_stack {
reqwest_with_middleware_builder = reqwest_with_middleware_builder.with_arc(m.clone());
}
let reqwest_with_middleware = reqwest_with_middleware_builder.build();
Ok(Self {
client: reqwest_with_middleware,
middleware_stack,
config,
})
}
}
impl Default for HttpClient {
fn default() -> Self {
HttpClient::from_config(Default::default()).expect("Failed to create default HttpClient")
}
}
impl HttpClientTrait for HttpClient {}

9
apps/recorder/src/fetch/client/mod.rs Normal file
View File

@@ -0,0 +1,9 @@
pub mod core;
pub mod secrecy;
pub use core::{
HttpClient, HttpClientCacheBackendConfig, HttpClientCachePresetConfig, HttpClientConfig,
HttpClientError, HttpClientTrait,
};
pub use secrecy::{HttpClientCookiesAuth, HttpClientSecrecyDataTrait};

55
apps/recorder/src/fetch/client/secrecy.rs Normal file
View File

@@ -0,0 +1,55 @@
use std::sync::Arc;
use cookie::Cookie;
use reqwest::{ClientBuilder, cookie::Jar};
use secrecy::zeroize::Zeroize;
use url::Url;
use crate::errors::RecorderError;
pub trait HttpClientSecrecyDataTrait: Zeroize {
fn attach_secrecy_to_client(&self, client_builder: ClientBuilder) -> ClientBuilder {
client_builder
}
}
#[derive(Default)]
pub struct HttpClientCookiesAuth {
pub cookie_jar: Arc<Jar>,
pub user_agent: Option<String>,
}
impl HttpClientCookiesAuth {
pub fn from_cookies(
cookies: &str,
url: &Url,
user_agent: Option<String>,
) -> Result<Self, RecorderError> {
let cookie_jar = Arc::new(Jar::default());
for cookie in Cookie::split_parse(cookies).try_collect::<Vec<_>>()? {
cookie_jar.add_cookie_str(&cookie.to_string(), url);
}
Ok(Self {
cookie_jar,
user_agent,
})
}
}
impl Zeroize for HttpClientCookiesAuth {
fn zeroize(&mut self) {
self.cookie_jar = Arc::new(Jar::default());
self.user_agent = None;
}
}
impl HttpClientSecrecyDataTrait for HttpClientCookiesAuth {
fn attach_secrecy_to_client(&self, client_builder: ClientBuilder) -> ClientBuilder {
let mut client_builder = client_builder.cookie_provider(self.cookie_jar.clone());
if let Some(ref user_agent) = self.user_agent {
client_builder = client_builder.user_agent(user_agent);
}
client_builder
}
}

3
apps/recorder/src/fetch/html.rs
View File

@@ -1,11 +1,12 @@
use reqwest::IntoUrl;
use super::client::HttpClientTrait;
use crate::errors::RecorderError;
pub async fn fetch_html<T: IntoUrl, H: HttpClientTrait>(
client: &H,
url: T,
) -> color_eyre::eyre::Result<String> {
) -> Result<String, RecorderError> {
let content = client
.get(url)
.send()

3
apps/recorder/src/fetch/image.rs
View File

@@ -2,10 +2,11 @@ use bytes::Bytes;
use reqwest::IntoUrl;
use super::{bytes::fetch_bytes, client::HttpClientTrait};
use crate::errors::RecorderError;
pub async fn fetch_image<T: IntoUrl, H: HttpClientTrait>(
client: &H,
url: T,
) -> color_eyre::eyre::Result<Bytes> {
) -> Result<Bytes, RecorderError> {
fetch_bytes(client, url).await
}

5
apps/recorder/src/fetch/mod.rs
View File

@@ -8,6 +8,9 @@ pub mod oidc;
pub use core::get_random_mobile_ua;
pub use bytes::fetch_bytes;
pub use client::{HttpClient, HttpClientConfig, HttpClientError, HttpClientTrait};
pub use client::{
HttpClient, HttpClientConfig, HttpClientCookiesAuth, HttpClientError,
HttpClientSecrecyDataTrait, HttpClientTrait,
};
pub use html::fetch_html;
pub use image::fetch_image;

4
apps/recorder/src/lib.rs
View File

@@ -2,7 +2,8 @@
duration_constructors,
assert_matches,
unboxed_closures,
impl_trait_in_bindings
impl_trait_in_bindings,
iterator_try_collect
)]
pub mod app;
@@ -10,6 +11,7 @@ pub mod auth;
pub mod config;
pub mod controllers;
pub mod dal;
pub mod errors;
pub mod extract;
pub mod fetch;
pub mod graphql;

2
apps/recorder/src/models/episodes.rs
View File

@@ -203,7 +203,7 @@ impl ActiveModel {
let homepage = build_mikan_episode_homepage(
ctx.get_mikan_client().base_url().clone(),
&item.mikan_episode_id,
)?;
);
Ok(Self {
mikan_episode_id: ActiveValue::Set(Some(item.mikan_episode_id)),

2
apps/recorder/src/models/subscriptions.rs
View File

@@ -274,7 +274,7 @@ impl Model {
mikan_base_url.clone(),
&mikan_bangumi_id,
Some(&mikan_fansub_id),
)?;
);
let bgm_rss_link = build_mikan_bangumi_rss_link(
mikan_base_url.clone(),
&mikan_bangumi_id,

28
apps/recorder/src/tasks/create_mikan_bangumi_subscriptions_from_my_bangumi_page.rs
View File

@@ -1,11 +1,17 @@
use loco_rs::prelude::*;
use secrecy::SecretString;
#[derive(Clone, Debug)]
use crate::{
app::AppContextExt,
extract::mikan::{
MikanAuthSecrecy, web_extract::extract_mikan_bangumis_meta_from_my_bangumi_page,
},
};
#[derive(Debug)]
pub struct CreateMikanRSSFromMyBangumiTask {
subscriber_id: i32,
task_id: String,
cookie: SecretString,
auth_secrecy: MikanAuthSecrecy,
}
#[async_trait::async_trait]
@@ -20,7 +26,21 @@ impl Task for CreateMikanRSSFromMyBangumiTask {
}
}
async fn run(&self, _app_context: &AppContext, _vars: &task::Vars) -> Result<()> {
async fn run(&self, app_context: &AppContext, _vars: &task::Vars) -> Result<()> {
let mikan_client = app_context
.get_mikan_client()
.fork_with_auth(self.auth_secrecy.clone())?;
// TODO
let _bangumi_metas = extract_mikan_bangumis_meta_from_my_bangumi_page(
&mikan_client,
mikan_client
.base_url()
.join("/Home/MyBangumi")
.map_err(loco_rs::Error::wrap)?,
)
.await?;
Ok(())
}
}

8
apps/recorder/src/test_utils/tracing.rs
View File

@@ -3,10 +3,8 @@ use tracing_subscriber::EnvFilter;
pub fn init_testing_tracing(level: Level) {
let crate_name = env!("CARGO_PKG_NAME");
let filter = EnvFilter::new(format!(
"{}[]={}",
crate_name,
level.as_str().to_lowercase()
));
let level = level.as_str().to_lowercase();
let filter = EnvFilter::new(format!("{}[]={}", crate_name, level))
.add_directive(format!("mockito[]={}", level).parse().unwrap());
tracing_subscriber::fmt().with_env_filter(filter).init();
}