dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fix subscriptions api

Browse Source
This commit is contained in:
master
2025-05-10 02:31:58 +08:00
parent d2aab7369d
commit 8144986a48
42 changed files with 815 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/proxy/.whistle/rules/files/1.mikan_doppel Normal file
View File

@@ -0,0 +1 @@
^https://mikanani.me/*** http://127.0.0.1:5010/$1

2
apps/proxy/.whistle/rules/properties
View File

@@ -1 +1 @@
{"filesOrder":["konobangu"],"selectedList":["konobangu"],"disabledDefalutRules":true,"defalutRules":""}
{"filesOrder":["konobangu","mikan_doppel"],"selectedList":["konobangu","mikan_doppel"],"disabledDefalutRules":true,"defalutRules":""}

19
apps/proxy/Cargo.toml Normal file
View File

@@ -0,0 +1,19 @@
[package]
name = "proxy"
version = "0.1.0"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "proxy"
path = "src/lib.rs"
[[bin]]
name = "mikan_doppel"
path = "src/bin/mikan_doppel.rs"
[dependencies]
recorder = { workspace = true }
tokio = { workspace = true }
tracing-subscriber = { workspace = true }
tracing = { workspace = true }

5
apps/proxy/package.json
View File

@@ -3,8 +3,9 @@
"version": "0.1.0",
"private": true,
"scripts": {
"start": "cross-env WHISTLE_MODE=\"prod|capture|keepXFF|x-forwarded-host|x-forwarded-proto\" whistle run -p 8899 -t 30000 -D .",
"dev": "pnpm run start"
"whistle": "cross-env WHISTLE_MODE=\"prod|capture|keepXFF|x-forwarded-host|x-forwarded-proto\" whistle run -p 8899 -t 30000 -D .",
"mikan_doppel": "cargo run -p proxy --bin mikan_doppel",
"dev": "npm-run-all -p mikan_doppel whistle"
},
"keywords": [],
"license": "MIT",

22
apps/proxy/src/bin/mikan_doppel.rs Normal file
View File

@@ -0,0 +1,22 @@
use std::time::Duration;
use recorder::{errors::RecorderResult, test_utils::mikan::MikanMockServer};
use tracing::Level;
#[allow(unused_variables)]
#[tokio::main]
async fn main() -> RecorderResult<()> {
tracing_subscriber::fmt()
.with_max_level(Level::DEBUG)
.init();
let mut mikan_server = MikanMockServer::new_with_port(5010).await.unwrap();
let resources_mock = mikan_server.mock_resources_with_doppel();
let login_mock = mikan_server.mock_get_login_page();
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
}
}

0
apps/proxy/src/lib.rs Normal file
View File

6
apps/recorder/Cargo.toml
View File

@@ -15,7 +15,7 @@ required-features = []
[features]
default = []
playground = ["dep:mockito", "dep:inquire", "dep:color-eyre"]
playground = ["dep:inquire", "dep:color-eyre"]
testcontainers = [
"dep:testcontainers",
"dep:testcontainers-modules",
@@ -54,7 +54,7 @@ serde_with = { workspace = true }
moka = { workspace = true }
chrono = { workspace = true }
tracing-subscriber = { workspace = true }
mockito = { workspace = true, optional = true }
mockito = { workspace = true }
sea-orm = { version = "1.1", features = [
"sqlx-sqlite",
@@ -122,11 +122,11 @@ color-eyre = { workspace = true, optional = true }
inquire = { workspace = true, optional = true }
percent-encoding = "2.3.1"
[dev-dependencies]
serial_test = "3"
insta = { version = "1", features = ["redactions", "toml", "filters"] }
rstest = "0.25"
ctor = "0.4.0"
mockito = { workspace = true }
inquire = { workspace = true }
color-eyre = { workspace = true }

6
apps/recorder/examples/mikan_doppel_season_subscription.rs
View File

@@ -6,7 +6,7 @@ use inquire::{Password, Text, validator::Validation};
use recorder::{
crypto::UserPassCredential,
extract::mikan::{
MikanClient, MikanConfig, MikanRssItem, build_mikan_bangumi_expand_subscribed_url,
MikanClient, MikanConfig, MikanRssEpisodeItem, build_mikan_bangumi_expand_subscribed_url,
extract_mikan_bangumi_index_meta_list_from_season_flow_fragment,
extract_mikan_bangumi_meta_from_expand_subscribed_fragment,
},
@@ -193,12 +193,12 @@ async fn main() -> Result<()> {
let rss_items = rss::Channel::read_from(bangumi_rss_data.as_bytes())?.items;
rss_items
.into_iter()
.map(MikanRssItem::try_from)
.map(MikanRssEpisodeItem::try_from)
.collect::<Result<Vec<_>, _>>()
}?;
for rss_item in rss_items {
{
let episode_homepage_url = rss_item.homepage;
let episode_homepage_url = rss_item.build_homepage_url(mikan_base_url.clone());
let episode_homepage_doppel_path =
MikanDoppelPath::new(episode_homepage_url.clone());
tracing::info!(title = rss_item.title, "Scraping episode...");

8
apps/recorder/examples/mikan_doppel_subscriber_subscription.rs
View File

@@ -4,7 +4,7 @@ use fetch::{FetchError, HttpClientConfig, fetch_bytes, fetch_html, fetch_image,
use recorder::{
errors::RecorderResult,
extract::mikan::{
MikanClient, MikanConfig, MikanRssItem,
MikanClient, MikanConfig, MikanRssEpisodeItem,
extract_mikan_episode_meta_from_episode_homepage_html,
},
test_utils::mikan::{MikanDoppelMeta, MikanDoppelPath},
@@ -43,15 +43,15 @@ async fn main() -> RecorderResult<()> {
let subscriber_subscription =
fs::read("tests/resources/mikan/MyBangumi-2025-spring.rss").await?;
let channel = rss::Channel::read_from(&subscriber_subscription[..])?;
let rss_items: Vec<MikanRssItem> = channel
let rss_items: Vec<MikanRssEpisodeItem> = channel
.items
.into_iter()
.map(MikanRssItem::try_from)
.map(MikanRssEpisodeItem::try_from)
.collect::<Result<Vec<_>, _>>()?;
for rss_item in rss_items {
let episode_homepage_meta = {
tracing::info!(title = rss_item.title, "Scraping episode homepage...");
let episode_homepage_url = rss_item.homepage;
let episode_homepage_url = rss_item.build_homepage_url(mikan_base_url.clone());
let episode_homepage_doppel_path = MikanDoppelPath::new(episode_homepage_url.clone());
let episode_homepage_data = if !episode_homepage_doppel_path.exists_any() {
let episode_homepage_data =

40
apps/recorder/src/app/core.rs
View File

@@ -6,7 +6,7 @@ use tracing::instrument;
use super::{builder::AppBuilder, context::AppContextTrait};
use crate::{
errors::RecorderResult,
errors::{RecorderError, RecorderResult},
web::{
controller::{self, core::ControllerTrait},
middleware::default_middleware_stack,
@@ -71,12 +71,38 @@ impl App {
.with_state(context.clone())
.into_make_service_with_connect_info::<SocketAddr>();
axum::serve(listener, router)
.with_graceful_shutdown(async move {
Self::shutdown_signal().await;
tracing::info!("shutting down...");
})
.await?;
let task = context.task();
tokio::try_join!(
async {
axum::serve(listener, router)
.with_graceful_shutdown(async move {
Self::shutdown_signal().await;
tracing::info!("axum shutting down...");
})
.await?;
Ok::<(), RecorderError>(())
},
async {
let monitor = task.setup_monitor().await?;
monitor
.run_with_signal(async move {
Self::shutdown_signal().await;
tracing::info!("apalis shutting down...");
Ok(())
})
.await?;
Ok::<(), RecorderError>(())
},
async {
let listener = task.setup_listener().await?;
listener.listen().await?;
Ok::<(), RecorderError>(())
}
)?;
Ok(())
}

7
apps/recorder/src/database/service.rs
View File

@@ -11,14 +11,16 @@ use super::DatabaseConfig;
use crate::{errors::RecorderResult, migrations::Migrator};
pub struct DatabaseService {
pub config: DatabaseConfig,
connection: DatabaseConnection,
#[cfg(all(any(test, feature = "playground"), feature = "testcontainers"))]
#[cfg(feature = "testcontainers")]
pub container:
Option<testcontainers::ContainerAsync<testcontainers_modules::postgres::Postgres>>,
}
impl DatabaseService {
pub async fn from_config(config: DatabaseConfig) -> RecorderResult<Self> {
let db_config = config.clone();
let mut opt = ConnectOptions::new(&config.uri);
opt.max_connections(config.max_connections)
.min_connections(config.min_connections)
@@ -50,8 +52,9 @@ impl DatabaseService {
let me = Self {
connection: db,
#[cfg(all(any(test, feature = "playground"), feature = "testcontainers"))]
#[cfg(feature = "testcontainers")]
container: None,
config: db_config,
};
if config.auto_migrate {

2
apps/recorder/src/errors/app_error.rs
View File

@@ -78,7 +78,7 @@ pub enum RecorderError {
},
#[snafu(transparent)]
HttpClientError { source: HttpClientError },
#[cfg(all(any(test, feature = "playground"), feature = "testcontainers"))]
#[cfg(feature = "testcontainers")]
#[snafu(transparent)]
TestcontainersError {
source: testcontainers::TestcontainersError,

2
apps/recorder/src/extract/mikan/mod.rs
View File

@@ -22,7 +22,7 @@ pub use subscription::{
};
pub use web::{
MikanBangumiHash, MikanBangumiIndexHash, MikanBangumiIndexMeta, MikanBangumiMeta,
MikanBangumiPosterMeta, MikanEpisodeHash, MikanEpisodeMeta, MikanRssItem,
MikanBangumiPosterMeta, MikanEpisodeHash, MikanEpisodeMeta, MikanRssEpisodeItem,
MikanSeasonFlowUrlMeta, MikanSeasonStr, MikanSubscriberSubscriptionRssUrlMeta,
build_mikan_bangumi_expand_subscribed_url, build_mikan_bangumi_homepage_url,
build_mikan_bangumi_subscription_rss_url, build_mikan_episode_homepage_url,

305
apps/recorder/src/extract/mikan/subscription.rs
View File

@@ -20,10 +20,10 @@ use crate::{
app::AppContextTrait,
errors::{RecorderError, RecorderResult},
extract::mikan::{
MikanBangumiHash, MikanBangumiMeta, MikanEpisodeHash, MikanEpisodeMeta, MikanRssItem,
MikanSeasonFlowUrlMeta, MikanSeasonStr, MikanSubscriberSubscriptionRssUrlMeta,
build_mikan_bangumi_subscription_rss_url, build_mikan_season_flow_url,
build_mikan_subscriber_subscription_rss_url,
MikanBangumiHash, MikanBangumiMeta, MikanEpisodeHash, MikanEpisodeMeta,
MikanRssEpisodeItem, MikanSeasonFlowUrlMeta, MikanSeasonStr,
MikanSubscriberSubscriptionRssUrlMeta, build_mikan_bangumi_subscription_rss_url,
build_mikan_season_flow_url, build_mikan_subscriber_subscription_rss_url,
scrape_mikan_episode_meta_from_episode_homepage_url,
},
models::{
@@ -35,10 +35,11 @@ use crate::{
#[tracing::instrument(err, skip(ctx, rss_item_list))]
async fn sync_mikan_feeds_from_rss_item_list(
ctx: &dyn AppContextTrait,
rss_item_list: Vec<MikanRssItem>,
rss_item_list: Vec<MikanRssEpisodeItem>,
subscriber_id: i32,
subscription_id: i32,
) -> RecorderResult<()> {
let mikan_base_url = ctx.mikan().base_url().clone();
let (new_episode_meta_list, existed_episode_hash2id_map) = {
let existed_episode_hash2id_map = episodes::Model::get_existed_mikan_episode_list(
ctx,
@@ -60,7 +61,7 @@ async fn sync_mikan_feeds_from_rss_item_list(
}) {
let episode_meta = scrape_mikan_episode_meta_from_episode_homepage_url(
mikan_client,
to_insert_rss_item.homepage,
to_insert_rss_item.build_homepage_url(mikan_base_url.clone()),
)
.await?;
new_episode_meta_list.push(episode_meta);
@@ -215,7 +216,7 @@ impl MikanSubscriberSubscription {
async fn get_rss_item_list_from_source_url(
&self,
ctx: &dyn AppContextTrait,
) -> RecorderResult<Vec<MikanRssItem>> {
) -> RecorderResult<Vec<MikanRssEpisodeItem>> {
let mikan_base_url = ctx.mikan().base_url().clone();
let rss_url = build_mikan_subscriber_subscription_rss_url(
mikan_base_url.clone(),
@@ -227,7 +228,7 @@ impl MikanSubscriberSubscription {
let mut result = vec![];
for (idx, item) in channel.items.into_iter().enumerate() {
let item = MikanRssItem::try_from(item)
let item = MikanRssEpisodeItem::try_from(item)
.with_whatever_context::<_, String, RecorderError>(|_| {
format!("failed to extract rss item at idx {idx}")
})?;
@@ -240,7 +241,7 @@ impl MikanSubscriberSubscription {
async fn get_rss_item_list_from_subsribed_url_rss_link(
&self,
ctx: &dyn AppContextTrait,
) -> RecorderResult<Vec<MikanRssItem>> {
) -> RecorderResult<Vec<MikanRssEpisodeItem>> {
let subscribed_bangumi_list =
bangumi::Model::get_subsribed_bangumi_list_from_subscription(ctx, self.id).await?;
@@ -259,7 +260,7 @@ impl MikanSubscriberSubscription {
let channel = rss::Channel::read_from(&bytes[..])?;
for (idx, item) in channel.items.into_iter().enumerate() {
let item = MikanRssItem::try_from(item)
let item = MikanRssEpisodeItem::try_from(item)
.with_whatever_context::<_, String, RecorderError>(|_| {
format!("failed to extract rss item at idx {idx}")
})?;
@@ -395,7 +396,7 @@ impl MikanSeasonSubscription {
async fn get_rss_item_list_from_subsribed_url_rss_link(
&self,
ctx: &dyn AppContextTrait,
) -> RecorderResult<Vec<MikanRssItem>> {
) -> RecorderResult<Vec<MikanRssEpisodeItem>> {
let db = ctx.db();
let subscribed_bangumi_list = bangumi::Entity::find()
@@ -422,7 +423,7 @@ impl MikanSeasonSubscription {
let channel = rss::Channel::read_from(&bytes[..])?;
for (idx, item) in channel.items.into_iter().enumerate() {
let item = MikanRssItem::try_from(item)
let item = MikanRssEpisodeItem::try_from(item)
.with_whatever_context::<_, String, RecorderError>(|_| {
format!("failed to extract rss item at idx {idx}")
})?;
@@ -499,7 +500,7 @@ impl MikanBangumiSubscription {
async fn get_rss_item_list_from_source_url(
&self,
ctx: &dyn AppContextTrait,
) -> RecorderResult<Vec<MikanRssItem>> {
) -> RecorderResult<Vec<MikanRssEpisodeItem>> {
let mikan_base_url = ctx.mikan().base_url().clone();
let rss_url = build_mikan_bangumi_subscription_rss_url(
mikan_base_url.clone(),
@@ -512,7 +513,7 @@ impl MikanBangumiSubscription {
let mut result = vec![];
for (idx, item) in channel.items.into_iter().enumerate() {
let item = MikanRssItem::try_from(item)
let item = MikanRssEpisodeItem::try_from(item)
.with_whatever_context::<_, String, RecorderError>(|_| {
format!("failed to extract rss item at idx {idx}")
})?;
@@ -522,106 +523,216 @@ impl MikanBangumiSubscription {
}
}
// #[cfg(test)]
// mod tests {
// use std::assert_matches::assert_matches;
#[cfg(test)]
#[allow(unused_variables)]
mod tests {
use std::sync::Arc;
// use downloader::bittorrent::BITTORRENT_MIME_TYPE;
// use rstest::rstest;
// use url::Url;
use rstest::{fixture, rstest};
use sea_orm::{ActiveModelTrait, ActiveValue, EntityTrait};
use tracing::Level;
// use crate::{
// errors::RecorderResult,
// extract::mikan::{
// MikanBangumiIndexRssChannel, MikanBangumiRssChannel,
// MikanRssChannel, build_mikan_bangumi_subscription_rss_url,
// extract_mikan_rss_channel_from_rss_link, },
// test_utils::mikan::build_testing_mikan_client,
// };
use crate::{
app::AppContextTrait,
errors::RecorderResult,
extract::mikan::{
MikanBangumiHash, MikanSeasonFlowUrlMeta, MikanSeasonStr,
MikanSubscriberSubscriptionRssUrlMeta,
},
models::{
bangumi,
subscriptions::{self, SubscriptionTrait},
},
test_utils::{
app::TestingAppContext,
crypto::build_testing_crypto_service,
database::build_testing_database_service,
mikan::{
MikanMockServer, build_testing_mikan_client, build_testing_mikan_credential_form,
},
storage::build_testing_storage_service,
tracing::try_init_testing_tracing,
},
};
// #[rstest]
// #[tokio::test]
// async fn test_parse_mikan_rss_channel_from_rss_link() ->
// RecorderResult<()> { let mut mikan_server =
// mockito::Server::new_async().await;
struct TestingResources {
pub app_ctx: Arc<dyn AppContextTrait>,
pub mikan_server: MikanMockServer,
}
// let mikan_base_url = Url::parse(&mikan_server.url())?;
async fn build_testing_app_context() -> RecorderResult<TestingResources> {
let mikan_server = MikanMockServer::new().await?;
// let mikan_client =
// build_testing_mikan_client(mikan_base_url.clone()).await?;
let mikan_base_url = mikan_server.base_url().clone();
// {
// let bangumi_rss_url = build_mikan_bangumi_subscription_rss_url(
// mikan_base_url.clone(),
// "3141",
// Some("370"),
// );
let app_ctx = {
let mikan_client = build_testing_mikan_client(mikan_base_url.clone()).await?;
let db_service = build_testing_database_service(Default::default()).await?;
let crypto_service = build_testing_crypto_service().await?;
let storage_service = build_testing_storage_service().await?;
let app_ctx = TestingAppContext::builder()
.mikan(mikan_client)
.db(db_service)
.crypto(crypto_service)
.storage(storage_service)
.build();
// let bangumi_rss_mock = mikan_server
// .mock("GET", bangumi_rss_url.path())
//
// .with_body_from_file("tests/resources/mikan/Bangumi-3141-370.rss")
// .match_query(mockito::Matcher::Any)
// .create_async()
// .await;
Arc::new(app_ctx)
};
// let channel =
// scrape_mikan_rss_channel_from_rss_link(&mikan_client, bangumi_rss_url)
// .await
// .expect("should get mikan channel from rss url");
Ok(TestingResources {
app_ctx,
mikan_server,
})
}
// assert_matches!(
// &channel,
// MikanRssChannel::Bangumi(MikanBangumiRssChannel { .. })
// );
#[fixture]
fn before_each() {
try_init_testing_tracing(Level::DEBUG);
}
// assert_matches!(&channel.name(), Some("葬送的芙莉莲"));
#[rstest]
#[tokio::test]
async fn test_mikan_season_subscription_sync_feeds(before_each: ()) -> RecorderResult<()> {
let TestingResources {
app_ctx,
mut mikan_server,
} = build_testing_app_context().await?;
// let items = channel.items();
// let first_sub_item = items
// .first()
// .expect("mikan subscriptions should have at least one subs");
let _resources_mock = mikan_server.mock_resources_with_doppel();
// assert_eq!(first_sub_item.mime, BITTORRENT_MIME_TYPE);
let _login_mock = mikan_server.mock_get_login_page();
// assert!(
// &first_sub_item
// .homepage
// .as_str()
// .starts_with("https://mikanani.me/Home/Episode")
// );
let mikan_client = app_ctx.mikan();
// let name = first_sub_item.title.as_str();
// assert!(name.contains("葬送的芙莉莲"));
let subscriber_id = 1;
// bangumi_rss_mock.expect(1);
// }
// {
// let bangumi_rss_url =
// mikan_base_url.join("/RSS/Bangumi?bangumiId=3416")?;
let credential = mikan_client
.submit_credential_form(
app_ctx.as_ref(),
subscriber_id,
build_testing_mikan_credential_form(),
)
.await?;
// let bangumi_rss_mock = mikan_server
// .mock("GET", bangumi_rss_url.path())
// .match_query(mockito::Matcher::Any)
//
// .with_body_from_file("tests/resources/mikan/Bangumi-3416.rss")
// .create_async()
// .await;
let subscription_am = subscriptions::ActiveModel {
display_name: ActiveValue::Set("test subscription".to_string()),
subscriber_id: ActiveValue::Set(subscriber_id),
category: ActiveValue::Set(subscriptions::SubscriptionCategory::MikanSeason),
source_url: ActiveValue::Set(
MikanSeasonFlowUrlMeta {
year: 2025,
season_str: MikanSeasonStr::Spring,
}
.build_season_flow_url(mikan_server.base_url().clone())
.to_string(),
),
enabled: ActiveValue::Set(true),
credential_id: ActiveValue::Set(Some(credential.id)),
..Default::default()
};
// let channel =
// scrape_mikan_rss_channel_from_rss_link(&mikan_client, bangumi_rss_url)
// .await
// .expect("should get mikan channel from rss url");
let subscription_model = subscription_am.insert(app_ctx.db()).await?;
// assert_matches!(
// &channel,
// MikanRssChannel::BangumiIndex(MikanBangumiIndexRssChannel {
// .. }) );
let subscription = subscriptions::Subscription::try_from_model(&subscription_model)?;
// assert_matches!(&channel.name(), Some("叹气的亡灵想隐退"));
{
subscription.sync_feeds_incremental(app_ctx.clone()).await?;
let bangumi_list = bangumi::Entity::find().all(app_ctx.db()).await?;
// bangumi_rss_mock.expect(1);
// }
// Ok(())
// }
// }
assert!(bangumi_list.is_empty());
}
{
subscription.sync_feeds_full(app_ctx.clone()).await?;
let bangumi_list = bangumi::Entity::find().all(app_ctx.db()).await?;
assert!(!bangumi_list.is_empty());
}
Ok(())
}
#[rstest]
#[tokio::test]
async fn test_mikan_subscriber_subscription_sync_feeds_incremental(
before_each: (),
) -> RecorderResult<()> {
let TestingResources {
app_ctx,
mut mikan_server,
} = build_testing_app_context().await?;
let _resources_mock = mikan_server.mock_resources_with_doppel();
let _login_mock = mikan_server.mock_get_login_page();
let subscriber_id = 1;
let subscription_am = subscriptions::ActiveModel {
display_name: ActiveValue::Set("test subscription".to_string()),
subscriber_id: ActiveValue::Set(subscriber_id),
category: ActiveValue::Set(subscriptions::SubscriptionCategory::MikanSubscriber),
source_url: ActiveValue::Set(
MikanSubscriberSubscriptionRssUrlMeta {
mikan_subscription_token: "123".into(),
}
.build_rss_url(mikan_server.base_url().clone())
.to_string(),
),
enabled: ActiveValue::Set(true),
..Default::default()
};
let subscription_model = subscription_am.insert(app_ctx.db()).await?;
let subscription_task = subscriptions::Subscription::try_from_model(&subscription_model)?;
subscription_task
.sync_feeds_incremental(app_ctx.clone())
.await?;
Ok(())
}
#[rstest]
#[tokio::test]
async fn test_mikan_bangumi_subscription_sync_feeds(before_each: ()) -> RecorderResult<()> {
let TestingResources {
app_ctx,
mut mikan_server,
} = build_testing_app_context().await?;
let _resources_mock = mikan_server.mock_resources_with_doppel();
let _login_mock = mikan_server.mock_get_login_page();
let subscriber_id = 1;
let subscription_am = subscriptions::ActiveModel {
display_name: ActiveValue::Set("test subscription".to_string()),
subscriber_id: ActiveValue::Set(subscriber_id),
category: ActiveValue::Set(subscriptions::SubscriptionCategory::MikanBangumi),
source_url: ActiveValue::Set(
MikanBangumiHash {
mikan_bangumi_id: "3600".into(),
mikan_fansub_id: "370".into(),
}
.build_rss_url(mikan_server.base_url().clone())
.to_string(),
),
enabled: ActiveValue::Set(true),
..Default::default()
};
let subscription_model = subscription_am.insert(app_ctx.db()).await?;
let subscription_task = subscriptions::Subscription::try_from_model(&subscription_model)?;
subscription_task
.sync_feeds_incremental(app_ctx.clone())
.await?;
Ok(())
}
}

23
apps/recorder/src/extract/mikan/web.rs
View File

@@ -32,9 +32,8 @@ use crate::{
};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MikanRssItem {
pub struct MikanRssEpisodeItem {
pub title: String,
pub homepage: Url,
pub url: Url,
pub content_length: Option<u64>,
pub mime: String,
@@ -42,7 +41,13 @@ pub struct MikanRssItem {
pub mikan_episode_id: String,
}
impl TryFrom<rss::Item> for MikanRssItem {
impl MikanRssEpisodeItem {
pub fn build_homepage_url(&self, mikan_base_url: Url) -> Url {
build_mikan_episode_homepage_url(mikan_base_url, &self.mikan_episode_id)
}
}
impl TryFrom<rss::Item> for MikanRssEpisodeItem {
type Error = RecorderError;
fn try_from(item: rss::Item) -> Result<Self, Self::Error> {
@@ -83,9 +88,8 @@ impl TryFrom<rss::Item> for MikanRssItem {
RecorderError::from_mikan_rss_invalid_field(Cow::Borrowed("mikan_episode_id"))
})?;
Ok(MikanRssItem {
Ok(MikanRssEpisodeItem {
title,
homepage,
url: enclosure_url,
content_length: enclosure.length.parse().ok(),
mime: mime_type,
@@ -436,6 +440,10 @@ impl MikanSeasonFlowUrlMeta {
None
}
}
pub fn build_season_flow_url(self, mikan_base_url: Url) -> Url {
build_mikan_season_flow_url(mikan_base_url, self.year, self.season_str)
}
}
pub fn build_mikan_bangumi_homepage_url(
mikan_base_url: Url,
@@ -511,6 +519,7 @@ pub fn extract_mikan_episode_meta_from_episode_homepage_html(
.select(&Selector::parse("title").unwrap())
.next()
.map(extract_inner_text_from_element_ref)
.map(|s| s.replace(" - Mikan Project", ""))
.ok_or_else(|| {
RecorderError::from_mikan_meta_missing_field(Cow::Borrowed("episode_title"))
})?;
@@ -543,7 +552,7 @@ pub fn extract_mikan_episode_meta_from_episode_homepage_html(
})
});
tracing::trace!(
tracing::debug!(
bangumi_title,
mikan_bangumi_id,
episode_title,
@@ -566,7 +575,7 @@ pub fn extract_mikan_episode_meta_from_episode_homepage_html(
})
}
#[instrument(skip_all, fields(mikan_episode_homepage_url = mikan_episode_homepage_url.as_str()))]
#[instrument(err, skip_all, fields(mikan_episode_homepage_url = mikan_episode_homepage_url.as_str()))]
pub async fn scrape_mikan_episode_meta_from_episode_homepage_url(
http_client: &MikanClient,
mikan_episode_homepage_url: Url,

2
apps/recorder/src/extract/rawname/mod.rs
View File

@@ -1,5 +1,5 @@
pub mod parser;
pub use parser::{
extract_season_from_title_body, parse_episode_meta_from_raw_name, RawEpisodeMeta,
RawEpisodeMeta, extract_episode_meta_from_raw_name, extract_season_from_title_body,
};

6
apps/recorder/src/extract/rawname/parser.rs
View File

@@ -261,7 +261,7 @@ pub fn check_is_movie(title: &str) -> bool {
MOVIE_TITLE_RE.is_match(title)
}
pub fn parse_episode_meta_from_raw_name(s: &str) -> RecorderResult<RawEpisodeMeta> {
pub fn extract_episode_meta_from_raw_name(s: &str) -> RecorderResult<RawEpisodeMeta> {
let raw_title = s.trim();
let raw_title_without_ch_brackets = replace_ch_bracket_to_en(raw_title);
let fansub = extract_fansub(&raw_title_without_ch_brackets);
@@ -321,11 +321,11 @@ pub fn parse_episode_meta_from_raw_name(s: &str) -> RecorderResult<RawEpisodeMet
#[cfg(test)]
mod tests {
use super::{RawEpisodeMeta, parse_episode_meta_from_raw_name};
use super::{RawEpisodeMeta, extract_episode_meta_from_raw_name};
fn test_raw_ep_parser_case(raw_name: &str, expected: &str) {
let expected: Option<RawEpisodeMeta> = serde_json::from_str(expected).unwrap_or_default();
let found = parse_episode_meta_from_raw_name(raw_name).ok();
let found = extract_episode_meta_from_raw_name(raw_name).ok();
if expected != found {
println!(

10
apps/recorder/src/graphql/views/subscription.rs
View File

@@ -11,7 +11,7 @@ use crate::{
app::AppContextTrait,
auth::AuthUserInfo,
models::subscriptions::{self, SubscriptionTrait},
task::SubscriberTaskPayload,
task::SubscriberTask,
};
#[derive(DynamicGraphql, Serialize, Deserialize, Clone, Debug)]
@@ -106,7 +106,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let task_id = task_service
.add_subscriber_task(
auth_user_info.subscriber_auth.subscriber_id,
SubscriberTaskPayload::SyncOneSubscriptionFeedsIncremental(
SubscriberTask::SyncOneSubscriptionFeedsIncremental(
subscription.into(),
),
)
@@ -156,9 +156,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let task_id = task_service
.add_subscriber_task(
auth_user_info.subscriber_auth.subscriber_id,
SubscriberTaskPayload::SyncOneSubscriptionFeedsFull(
subscription.into(),
),
SubscriberTask::SyncOneSubscriptionFeedsFull(subscription.into()),
)
.await?;
@@ -206,7 +204,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let task_id = task_service
.add_subscriber_task(
auth_user_info.subscriber_auth.subscriber_id,
SubscriberTaskPayload::SyncOneSubscriptionSources(subscription.into()),
SubscriberTask::SyncOneSubscriptionSources(subscription.into()),
)
.await?;

1
apps/recorder/src/lib.rs
View File

@@ -26,6 +26,5 @@ pub mod migrations;
pub mod models;
pub mod storage;
pub mod task;
#[cfg(any(test, feature = "playground"))]
pub mod test_utils;
pub mod web;

2
apps/recorder/src/logger/mod.rs
View File

@@ -5,4 +5,4 @@ pub mod service;
pub use core::{LogFormat, LogLevel, LogRotation};
pub use config::{LoggerConfig, LoggerFileAppender};
pub use service::LoggerService;
pub use service::{LoggerService, MODULE_WHITELIST};

2
apps/recorder/src/logger/service.rs
View File

@@ -13,7 +13,7 @@ use super::{LogFormat, LogLevel, LogRotation, LoggerConfig};
use crate::errors::RecorderResult;
// Function to initialize the logger based on the provided configuration
const MODULE_WHITELIST: &[&str] = &["sea_orm_migration", "tower_http", "sqlx::query", "sidekiq"];
pub const MODULE_WHITELIST: &[&str] = &["sea_orm_migration", "tower_http", "sea_orm", "sea_query"];
// Keep nonblocking file appender work guard
static NONBLOCKING_WORK_GUARD_KEEP: OnceLock<WorkerGuard> = OnceLock::new();

1
apps/recorder/src/migrations/defs.rs
View File

@@ -53,7 +53,6 @@ pub enum Bangumi {
PosterLink,
SavePath,
Homepage,
Extra,
}
#[derive(DeriveIden)]

32
apps/recorder/src/migrations/m20220101_000001_init.rs
View File

@@ -106,7 +106,6 @@ impl MigrationTrait for Migration {
.col(text_null(Bangumi::PosterLink))
.col(text_null(Bangumi::SavePath))
.col(text_null(Bangumi::Homepage))
.col(json_binary_null(Bangumi::Extra))
.foreign_key(
ForeignKey::create()
.name("fk_bangumi_subscriber_id")
@@ -209,7 +208,7 @@ impl MigrationTrait for Migration {
.create_index(
Index::create()
.if_not_exists()
.name("index_subscription_bangumi_subscriber_id")
.name("idx_subscription_bangumi_subscriber_id")
.table(SubscriptionBangumi::Table)
.col(SubscriptionBangumi::SubscriberId)
.to_owned(),
@@ -235,7 +234,6 @@ impl MigrationTrait for Migration {
.col(text_null(Episodes::Homepage))
.col(text_null(Episodes::Subtitle))
.col(text_null(Episodes::Source))
.col(json_binary_null(Episodes::Extra))
.foreign_key(
ForeignKey::create()
.name("fk_episodes_bangumi_id")
@@ -252,6 +250,15 @@ impl MigrationTrait for Migration {
.on_update(ForeignKeyAction::Cascade)
.on_delete(ForeignKeyAction::Cascade),
)
.index(
Index::create()
.if_not_exists()
.name("idx_episodes_mikan_episode_id_subscriber_id")
.table(Episodes::Table)
.col(Episodes::MikanEpisodeId)
.col(Episodes::SubscriberId)
.unique(),
)
.to_owned(),
)
.await?;
@@ -267,19 +274,6 @@ impl MigrationTrait for Migration {
)
.await?;
manager
.create_index(
Index::create()
.if_not_exists()
.name("idx_episodes_bangumi_id_mikan_episode_id")
.table(Episodes::Table)
.col(Episodes::BangumiId)
.col(Episodes::MikanEpisodeId)
.unique()
.to_owned(),
)
.await?;
manager
.create_postgres_auto_update_ts_trigger_for_col(Episodes::Table, GeneralIds::UpdatedAt)
.await?;
@@ -338,7 +332,7 @@ impl MigrationTrait for Migration {
.create_index(
Index::create()
.if_not_exists()
.name("index_subscription_episode_subscriber_id")
.name("idx_subscription_episode_subscriber_id")
.table(SubscriptionEpisode::Table)
.col(SubscriptionEpisode::SubscriberId)
.to_owned(),
@@ -353,7 +347,7 @@ impl MigrationTrait for Migration {
.drop_index(
Index::drop()
.if_exists()
.name("index_subscription_episode_subscriber_id")
.name("idx_subscription_episode_subscriber_id")
.table(SubscriptionBangumi::Table)
.to_owned(),
)
@@ -380,7 +374,7 @@ impl MigrationTrait for Migration {
.drop_index(
Index::drop()
.if_exists()
.name("index_subscription_bangumi_subscriber_id")
.name("idx_subscription_bangumi_subscriber_id")
.table(SubscriptionBangumi::Table)
.to_owned(),
)

16
apps/recorder/src/migrations/m20250520_021135_subscriber_tasks.rs
View File

@@ -35,14 +35,14 @@ AND jsonb_path_exists(job, '$.task_type ? (@.type() == "string")')"#,
))
.await?;
db.execute_unprepared(&format!(
r#"CREATE INDEX IF NOT EXISTS idx_apalis_jobs_subscriber_id
ON apalis.jobs ((job -> 'subscriber_id'))
WHERE job_type = '{SUBSCRIBER_TASK_APALIS_NAME}'
AND jsonb_path_exists(job, '$.subscriber_id ? (@.type() == "number")')
AND jsonb_path_exists(job, '$.task_type ? (@.type() == "string")')"#
))
.await?;
// db.execute_unprepared(&format!(
// r#"CREATE INDEX IF NOT EXISTS idx_apalis_jobs_subscriber_id
// ON apalis.jobs (((job -> 'subscriber_id')::integer))
// WHERE job_type = '{SUBSCRIBER_TASK_APALIS_NAME}'
// AND jsonb_path_exists(job, '$.subscriber_id ? (@.type() == "number")')
// AND jsonb_path_exists(job, '$.task_type ? (@.type() == "string")')"#
// ))
// .await?;
Ok(())
}

37
apps/recorder/src/models/bangumi.rs
View File

@@ -17,7 +17,7 @@ use crate::{
MikanBangumiHash, MikanBangumiMeta, build_mikan_bangumi_subscription_rss_url,
scrape_mikan_poster_meta_from_image_url,
},
rawname::parse_episode_meta_from_raw_name,
rawname::extract_season_from_title_body,
},
};
@@ -29,18 +29,6 @@ pub struct BangumiFilter {
pub group: Option<Vec<String>>,
}
#[derive(
Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult, SimpleObject,
)]
pub struct BangumiExtra {
pub name_zh: Option<String>,
pub s_name_zh: Option<String>,
pub name_en: Option<String>,
pub s_name_en: Option<String>,
pub name_jp: Option<String>,
pub s_name_jp: Option<String>,
}
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize, SimpleObject)]
#[sea_orm(table_name = "bangumi")]
pub struct Model {
@@ -63,7 +51,6 @@ pub struct Model {
pub poster_link: Option<String>,
pub save_path: Option<String>,
pub homepage: Option<String>,
pub extra: Option<BangumiExtra>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -135,8 +122,7 @@ impl ActiveModel {
let mikan_client = ctx.mikan();
let storage_service = ctx.storage();
let mikan_base_url = mikan_client.base_url();
let rawname_meta = parse_episode_meta_from_raw_name(&meta.bangumi_title)?;
let (_, season_raw, season_index) = extract_season_from_title_body(&meta.bangumi_title);
let rss_url = build_mikan_bangumi_subscription_rss_url(
mikan_base_url.clone(),
@@ -163,20 +149,12 @@ impl ActiveModel {
subscriber_id: ActiveValue::Set(subscriber_id),
display_name: ActiveValue::Set(meta.bangumi_title.clone()),
raw_name: ActiveValue::Set(meta.bangumi_title),
season: ActiveValue::Set(rawname_meta.season),
season_raw: ActiveValue::Set(rawname_meta.season_raw),
season: ActiveValue::Set(season_index),
season_raw: ActiveValue::Set(season_raw),
fansub: ActiveValue::Set(Some(meta.fansub)),
poster_link: ActiveValue::Set(poster_link),
homepage: ActiveValue::Set(Some(meta.homepage.to_string())),
rss_link: ActiveValue::Set(Some(rss_url.to_string())),
extra: ActiveValue::Set(Some(BangumiExtra {
name_zh: rawname_meta.name_zh,
name_en: rawname_meta.name_en,
name_jp: rawname_meta.name_jp,
s_name_en: rawname_meta.name_en_no_season,
s_name_jp: rawname_meta.name_jp_no_season,
s_name_zh: rawname_meta.name_zh_no_season,
})),
..Default::default()
})
}
@@ -218,15 +196,16 @@ impl Model {
Expr::col((
subscription_bangumi_alias.clone(),
subscription_bangumi::Column::SubscriptionId,
)),
))
.is_not_null(),
"is_subscribed",
)
.join_as_rev(
JoinType::LeftJoin,
subscription_bangumi::Relation::Bangumi
.def()
.on_condition(move |_left, right| {
Expr::col((right, subscription_bangumi::Column::SubscriptionId))
.on_condition(move |left, _right| {
Expr::col((left, subscription_bangumi::Column::SubscriptionId))
.eq(subscription_id)
.into_condition()
}),

79
apps/recorder/src/models/episodes.rs
View File

@@ -1,7 +1,6 @@
use async_trait::async_trait;
use sea_orm::{
ActiveValue, FromJsonQueryResult, IntoSimpleExpr, QuerySelect, entity::prelude::*,
sea_query::OnConflict,
ActiveValue, IntoSimpleExpr, QuerySelect, entity::prelude::*, sea_query::OnConflict,
};
use serde::{Deserialize, Serialize};
@@ -11,20 +10,10 @@ use crate::{
errors::RecorderResult,
extract::{
mikan::{MikanEpisodeHash, MikanEpisodeMeta, build_mikan_episode_homepage_url},
rawname::parse_episode_meta_from_raw_name,
rawname::extract_episode_meta_from_raw_name,
},
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult, Default)]
pub struct EpisodeExtra {
pub name_zh: Option<String>,
pub s_name_zh: Option<String>,
pub name_en: Option<String>,
pub s_name_en: Option<String>,
pub name_jp: Option<String>,
pub s_name_jp: Option<String>,
}
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
#[sea_orm(table_name = "episodes")]
pub struct Model {
@@ -50,7 +39,6 @@ pub struct Model {
pub homepage: Option<String>,
pub subtitle: Option<String>,
pub source: Option<String>,
pub extra: EpisodeExtra,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -135,42 +123,51 @@ impl ActiveModel {
episode: MikanEpisodeMeta,
) -> RecorderResult<Self> {
let mikan_base_url = ctx.mikan().base_url().clone();
let rawname_meta = parse_episode_meta_from_raw_name(&episode.episode_title)?;
let episode_extention_meta = extract_episode_meta_from_raw_name(&episode.episode_title)
.inspect_err(|err| {
tracing::error!(
err = ?err,
episode_title = ?episode.episode_title,
"Failed to parse episode extension meta from episode title, skip"
);
})
.ok();
let homepage = build_mikan_episode_homepage_url(mikan_base_url, &episode.mikan_episode_id);
Ok(Self {
let mut episode_active_model = Self {
mikan_episode_id: ActiveValue::Set(Some(episode.mikan_episode_id)),
raw_name: ActiveValue::Set(episode.episode_title.clone()),
display_name: ActiveValue::Set(episode.episode_title.clone()),
bangumi_id: ActiveValue::Set(bangumi.id),
subscriber_id: ActiveValue::Set(bangumi.subscriber_id),
resolution: ActiveValue::Set(rawname_meta.resolution),
season: ActiveValue::Set(if rawname_meta.season > 0 {
rawname_meta.season
} else {
bangumi.season
}),
season_raw: ActiveValue::Set(
rawname_meta
.season_raw
.or_else(|| bangumi.season_raw.clone()),
),
fansub: ActiveValue::Set(rawname_meta.fansub.or_else(|| bangumi.fansub.clone())),
poster_link: ActiveValue::Set(bangumi.poster_link.clone()),
episode_index: ActiveValue::Set(rawname_meta.episode_index),
homepage: ActiveValue::Set(Some(homepage.to_string())),
subtitle: ActiveValue::Set(rawname_meta.subtitle),
source: ActiveValue::Set(rawname_meta.source),
extra: ActiveValue::Set(EpisodeExtra {
name_zh: rawname_meta.name_zh,
name_en: rawname_meta.name_en,
name_jp: rawname_meta.name_jp,
s_name_en: rawname_meta.name_en_no_season,
s_name_jp: rawname_meta.name_jp_no_season,
s_name_zh: rawname_meta.name_zh_no_season,
}),
season_raw: ActiveValue::Set(bangumi.season_raw.clone()),
season: ActiveValue::Set(bangumi.season),
fansub: ActiveValue::Set(bangumi.fansub.clone()),
poster_link: ActiveValue::Set(bangumi.poster_link.clone()),
episode_index: ActiveValue::Set(0),
..Default::default()
})
};
if let Some(episode_extention_meta) = episode_extention_meta {
episode_active_model.episode_index =
ActiveValue::Set(episode_extention_meta.episode_index);
episode_active_model.subtitle = ActiveValue::Set(episode_extention_meta.subtitle);
episode_active_model.source = ActiveValue::Set(episode_extention_meta.source);
episode_active_model.resolution = ActiveValue::Set(episode_extention_meta.resolution);
if episode_extention_meta.season > 0 {
episode_active_model.season = ActiveValue::Set(episode_extention_meta.season);
}
if episode_extention_meta.season_raw.is_some() {
episode_active_model.season_raw =
ActiveValue::Set(episode_extention_meta.season_raw);
}
if episode_extention_meta.fansub.is_some() {
episode_active_model.fansub = ActiveValue::Set(episode_extention_meta.fansub);
}
}
Ok(episode_active_model)
}
}

3
apps/recorder/src/task/mod.rs
View File

@@ -7,7 +7,6 @@ pub use core::{SUBSCRIBER_TASK_APALIS_NAME, SubscriberAsyncTaskTrait, Subscriber
pub use config::TaskConfig;
pub use registry::{
SubscriberTask, SubscriberTaskPayload, SyncOneSubscriptionFeedsIncrementalTask,
SyncOneSubscriptionSourcesTask,
SubscriberTask, SyncOneSubscriptionFeedsIncrementalTask, SyncOneSubscriptionSourcesTask,
};
pub use service::TaskService;

56
apps/recorder/src/task/registry/mod.rs
View File

@@ -12,6 +12,7 @@ use super::SubscriberAsyncTaskTrait;
use crate::{
app::AppContextTrait,
errors::{RecorderError, RecorderResult},
models::subscriptions::SubscriptionTrait,
};
#[derive(async_graphql::Enum, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Copy)]
@@ -27,9 +28,26 @@ pub enum SubscriberTaskType {
SyncOneSubscriptionSources,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
impl TryFrom<&SubscriberTask> for serde_json::Value {
type Error = RecorderError;
fn try_from(value: &SubscriberTask) -> Result<Self, Self::Error> {
let json_value = serde_json::to_value(value)?;
Ok(match json_value {
serde_json::Value::Object(mut map) => {
map.remove("task_type");
serde_json::Value::Object(map)
}
_ => {
unreachable!("subscriber task must be an json object");
}
})
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, FromJsonQueryResult)]
#[serde(tag = "task_type")]
pub enum SubscriberTaskPayload {
pub enum SubscriberTask {
#[serde(rename = "sync_one_subscription_feeds_incremental")]
SyncOneSubscriptionFeedsIncremental(SyncOneSubscriptionFeedsIncrementalTask),
#[serde(rename = "sync_one_subscription_feeds_full")]
@@ -38,7 +56,15 @@ pub enum SubscriberTaskPayload {
SyncOneSubscriptionSources(SyncOneSubscriptionSourcesTask),
}
impl SubscriberTaskPayload {
impl SubscriberTask {
pub fn get_subscriber_id(&self) -> i32 {
match self {
Self::SyncOneSubscriptionFeedsIncremental(task) => task.0.get_subscriber_id(),
Self::SyncOneSubscriptionFeedsFull(task) => task.0.get_subscriber_id(),
Self::SyncOneSubscriptionSources(task) => task.0.get_subscriber_id(),
}
}
pub async fn run(self, ctx: Arc<dyn AppContextTrait>) -> RecorderResult<()> {
match self {
Self::SyncOneSubscriptionFeedsIncremental(task) => task.run(ctx).await,
@@ -59,27 +85,3 @@ impl SubscriberTaskPayload {
}
}
}
impl TryFrom<&SubscriberTaskPayload> for serde_json::Value {
type Error = RecorderError;
fn try_from(value: &SubscriberTaskPayload) -> Result<Self, Self::Error> {
let json_value = serde_json::to_value(value)?;
Ok(match json_value {
serde_json::Value::Object(mut map) => {
map.remove("task_type");
serde_json::Value::Object(map)
}
_ => {
unreachable!("subscriber task payload must be an json object");
}
})
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, FromJsonQueryResult)]
pub struct SubscriberTask {
pub subscriber_id: i32,
#[serde(flatten)]
pub payload: SubscriberTaskPayload,
}

48
apps/recorder/src/task/service.rs
View File

@@ -1,19 +1,22 @@
use std::{ops::Deref, sync::Arc};
use apalis::prelude::*;
use apalis_sql::{Config, postgres::PostgresStorage};
use apalis_sql::{
Config,
postgres::{PgListen, PostgresStorage},
};
use tokio::sync::RwLock;
use crate::{
app::AppContextTrait,
errors::RecorderResult,
task::{SUBSCRIBER_TASK_APALIS_NAME, SubscriberTask, SubscriberTaskPayload, TaskConfig},
task::{SUBSCRIBER_TASK_APALIS_NAME, SubscriberTask, TaskConfig},
};
pub struct TaskService {
pub config: TaskConfig,
ctx: Arc<dyn AppContextTrait>,
pub subscriber_task_storage: Arc<RwLock<PostgresStorage<SubscriberTask>>>,
subscriber_task_storage: Arc<RwLock<PostgresStorage<SubscriberTask>>>,
}
impl TaskService {
@@ -23,15 +26,12 @@ impl TaskService {
) -> RecorderResult<Self> {
let pool = ctx.db().get_postgres_connection_pool().clone();
let storage_config = Config::new(SUBSCRIBER_TASK_APALIS_NAME);
let subscriber_task_storage = Arc::new(RwLock::new(PostgresStorage::new_with_config(
pool,
storage_config,
)));
let subscriber_task_storage = PostgresStorage::new_with_config(pool, storage_config);
Ok(Self {
config,
ctx,
subscriber_task_storage,
subscriber_task_storage: Arc::new(RwLock::new(subscriber_task_storage)),
})
}
@@ -41,19 +41,14 @@ impl TaskService {
) -> RecorderResult<()> {
let ctx = data.deref().clone();
job.payload.run(ctx).await
job.run(ctx).await
}
pub async fn add_subscriber_task(
&self,
subscriber_id: i32,
task_payload: SubscriberTaskPayload,
_subscriber_id: i32,
subscriber_task: SubscriberTask,
) -> RecorderResult<TaskId> {
let subscriber_task = SubscriberTask {
subscriber_id,
payload: task_payload,
};
let task_id = {
let mut storage = self.subscriber_task_storage.write().await;
storage.push(subscriber_task).await?.task_id
@@ -62,22 +57,27 @@ impl TaskService {
Ok(task_id)
}
pub async fn setup(&self) -> RecorderResult<()> {
pub async fn setup_monitor(&self) -> RecorderResult<Monitor> {
let monitor = Monitor::new();
let worker = WorkerBuilder::new(SUBSCRIBER_TASK_APALIS_NAME)
.catch_panic()
.enable_tracing()
.data(self.ctx.clone())
.backend({
let storage = self.subscriber_task_storage.read().await;
storage.clone()
})
.backend(self.subscriber_task_storage.read().await.clone())
.build_fn(Self::run_subscriber_task);
let monitor = monitor.register(worker);
Ok(monitor.register(worker))
}
monitor.run().await?;
pub async fn setup_listener(&self) -> RecorderResult<PgListen> {
let pool = self.ctx.db().get_postgres_connection_pool().clone();
let mut subscriber_task_listener = PgListen::new(pool).await?;
Ok(())
{
let mut subscriber_task_storage = self.subscriber_task_storage.write().await;
subscriber_task_listener.subscribe_with(&mut subscriber_task_storage);
}
Ok(subscriber_task_listener)
}
}

4
apps/recorder/src/test_utils/app.rs
View File

@@ -3,7 +3,7 @@ use std::{fmt::Debug, sync::Arc};
use once_cell::sync::OnceCell;
use typed_builder::TypedBuilder;
use crate::app::AppContextTrait;
use crate::{app::AppContextTrait, test_utils::storage::TestingStorageService};
#[derive(TypedBuilder)]
#[builder(field_defaults(default, setter(strip_option)))]
@@ -15,7 +15,7 @@ pub struct TestingAppContext {
mikan: Option<crate::extract::mikan::MikanClient>,
auth: Option<crate::auth::AuthService>,
graphql: Option<crate::graphql::GraphQLService>,
storage: Option<crate::storage::StorageService>,
storage: Option<TestingStorageService>,
crypto: Option<crate::crypto::CryptoService>,
#[builder(default = Arc::new(OnceCell::new()), setter(!strip_option))]
task: Arc<OnceCell<crate::task::TaskService>>,

88
apps/recorder/src/test_utils/mikan.rs
View File

@@ -1,10 +1,12 @@
use std::{
collections::HashMap,
path::{self, Path},
ops::{Deref, DerefMut},
path::{self, PathBuf},
};
use chrono::{Duration, Utc};
use fetch::{FetchError, HttpClientConfig, IntoUrl, get_random_ua};
use lazy_static::lazy_static;
use percent_encoding::{AsciiSet, CONTROLS, percent_decode, utf8_percent_encode};
use serde::{Deserialize, Serialize};
use url::Url;
@@ -43,9 +45,7 @@ pub async fn build_testing_mikan_client(
base_mikan_url: impl IntoUrl,
) -> RecorderResult<MikanClient> {
let mikan_client = MikanClient::from_config(MikanConfig {
http_client: HttpClientConfig {
..Default::default()
},
http_client: HttpClientConfig::default(),
base_url: base_mikan_url.into_url().map_err(FetchError::from)?,
})
.await?;
@@ -147,10 +147,19 @@ impl AsRef<path::Path> for MikanDoppelPath {
}
}
lazy_static! {
static ref TEST_RESOURCES_DIR: String =
if cfg!(any(test, debug_assertions, feature = "playground")) {
format!("{}/tests/resources", env!("CARGO_MANIFEST_DIR"))
} else {
"tests/resources".to_string()
};
}
impl From<Url> for MikanDoppelPath {
fn from(value: Url) -> Self {
let base_path =
Path::new("tests/resources/mikan/doppel").join(value.path().trim_matches('/'));
let doppel_path = PathBuf::from(format!("{}/mikan/doppel", TEST_RESOURCES_DIR.as_str()));
let base_path = doppel_path.join(value.path().trim_matches('/'));
let dirname = base_path.parent();
let stem = base_path.file_stem();
debug_assert!(dirname.is_some() && stem.is_some());
@@ -187,17 +196,60 @@ pub struct MikanMockServerResourcesMock {
pub season_flow_noauth_mock: mockito::Mock,
}
pub enum MikanMockServerInner {
Server(mockito::Server),
ServerGuard(mockito::ServerGuard),
}
impl Deref for MikanMockServerInner {
type Target = mockito::Server;
fn deref(&self) -> &Self::Target {
match self {
MikanMockServerInner::Server(server) => server,
MikanMockServerInner::ServerGuard(server) => server,
}
}
}
impl DerefMut for MikanMockServerInner {
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
MikanMockServerInner::Server(server) => server,
MikanMockServerInner::ServerGuard(server) => server,
}
}
}
pub struct MikanMockServer {
pub server: mockito::ServerGuard,
pub server: MikanMockServerInner,
base_url: Url,
}
impl MikanMockServer {
pub async fn new_with_port(port: u16) -> RecorderResult<Self> {
let server = mockito::Server::new_with_opts_async(mockito::ServerOpts {
host: "0.0.0.0",
port,
..Default::default()
})
.await;
let base_url = Url::parse(&server.url())?;
Ok(Self {
server: MikanMockServerInner::Server(server),
base_url,
})
}
pub async fn new() -> RecorderResult<Self> {
let server = mockito::Server::new_async().await;
let base_url = Url::parse(&server.url())?;
Ok(Self { server, base_url })
Ok(Self {
server: MikanMockServerInner::ServerGuard(server),
base_url,
})
}
pub fn base_url(&self) -> &Url {
@@ -230,7 +282,10 @@ impl MikanMockServer {
SameSite=Strict; Path=/"
),
)
.with_body_from_file("tests/resources/mikan/LoginPage.html")
.with_body_from_file(format!(
"{}/mikan/LoginPage.html",
TEST_RESOURCES_DIR.as_str()
))
.create();
let test_identity_expires = (Utc::now() + Duration::days(30)).to_rfc2822();
@@ -284,7 +339,10 @@ impl MikanMockServer {
.match_query(mockito::Matcher::Any)
.match_request(move |req| !match_post_login_body(req))
.with_status(200)
.with_body_from_file("tests/resources/mikan/LoginError.html")
.with_body_from_file(format!(
"{}/mikan/LoginError.html",
TEST_RESOURCES_DIR.as_str()
))
.create();
let account_get_success_mock = self
@@ -428,7 +486,10 @@ impl MikanMockServer {
.starts_with(MIKAN_BANGUMI_EXPAND_SUBSCRIBED_PAGE_PATH)
})
.with_status(200)
.with_body_from_file("tests/resources/mikan/ExpandBangumi-noauth.html")
.with_body_from_file(format!(
"{}/mikan/ExpandBangumi-noauth.html",
TEST_RESOURCES_DIR.as_str()
))
.create();
let season_flow_noauth_mock = self
@@ -439,7 +500,10 @@ impl MikanMockServer {
&& req.path().starts_with(MIKAN_SEASON_FLOW_PAGE_PATH)
})
.with_status(200)
.with_body_from_file("tests/resources/mikan/BangumiCoverFlow-noauth.html")
.with_body_from_file(format!(
"{}/mikan/BangumiCoverFlow-noauth.html",
TEST_RESOURCES_DIR.as_str()
))
.create();
MikanMockServerResourcesMock {

14
apps/recorder/src/test_utils/tracing.rs
View File

@@ -1,11 +1,19 @@
use tracing::Level;
use tracing_subscriber::EnvFilter;
use crate::logger::MODULE_WHITELIST;
pub fn try_init_testing_tracing(level: Level) {
let crate_name = env!("CARGO_PKG_NAME");
let level = level.as_str().to_lowercase();
let filter = EnvFilter::new(format!("{crate_name}[]={level}"))
.add_directive(format!("mockito[]={level}").parse().unwrap())
.add_directive(format!("sqlx[]={level}").parse().unwrap());
let mut filter = EnvFilter::new(format!("{crate_name}[]={level}"));
let mut modules = vec![];
modules.extend(MODULE_WHITELIST.iter());
modules.push("mockito");
for module in modules {
filter = filter.add_directive(format!("{module}[]={level}").parse().unwrap());
}
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}

20
apps/webui/src/components/ui/data-table-row-actions.tsx
View File

@@ -1,9 +1,9 @@
'use client';
"use client";
import type { Row } from '@tanstack/react-table';
import { MoreHorizontal } from 'lucide-react';
import type { Row } from "@tanstack/react-table";
import { MoreHorizontal } from "lucide-react";
import { Button } from '@/components/ui/button';
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
@@ -11,9 +11,9 @@ import {
DropdownMenuSeparator,
DropdownMenuShortcut,
DropdownMenuTrigger,
} from '@/components/ui/dropdown-menu';
} from "@/components/ui/dropdown-menu";
import { useMemo } from 'react';
import { PropsWithChildren, useMemo } from "react";
interface DataTableRowActionsProps<DataView, Id> {
row: Row<DataView>;
@@ -24,6 +24,7 @@ interface DataTableRowActionsProps<DataView, Id> {
onDetail?: (id: Id) => void;
onDelete?: (id: Id) => void;
onEdit?: (id: Id) => void;
modal?: boolean;
}
export function DataTableRowActions<DataView, Id>({
@@ -35,10 +36,12 @@ export function DataTableRowActions<DataView, Id>({
onDetail,
onDelete,
onEdit,
}: DataTableRowActionsProps<DataView, Id>) {
children,
modal,
}: PropsWithChildren<DataTableRowActionsProps<DataView, Id>>) {
const id = useMemo(() => getId(row), [getId, row]);
return (
<DropdownMenu>
<DropdownMenu modal={modal}>
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
@@ -49,6 +52,7 @@ export function DataTableRowActions<DataView, Id>({
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-[160px]">
{children}
{showDetail && (
<DropdownMenuItem onClick={() => onDetail?.(id)}>
Detail

17
apps/webui/src/presentation/routes/_app/credential3rd/manage.tsx
View File

@@ -3,6 +3,8 @@ import { Button } from '@/components/ui/button';
import { DataTablePagination } from '@/components/ui/data-table-pagination';
import { DataTableRowActions } from '@/components/ui/data-table-row-actions';
import { DataTableViewOptions } from '@/components/ui/data-table-view-options';
import { DialogTrigger } from '@/components/ui/dialog';
import { DropdownMenuItem } from '@/components/ui/dropdown-menu';
import { QueryErrorView } from '@/components/ui/query-error-view';
import { Skeleton } from '@/components/ui/skeleton';
import {
@@ -28,6 +30,7 @@ import { useDebouncedSkeleton } from '@/presentation/hooks/use-debounded-skeleto
import { useEvent } from '@/presentation/hooks/use-event';
import { cn } from '@/presentation/utils';
import { useMutation, useQuery } from '@apollo/client';
import { Dialog } from '@radix-ui/react-dialog';
import { createFileRoute, useNavigate } from '@tanstack/react-router';
import {
type ColumnDef,
@@ -44,6 +47,7 @@ import { format } from 'date-fns';
import { Eye, EyeOff, Plus } from 'lucide-react';
import { useMemo, useState } from 'react';
import { toast } from 'sonner';
import { Credential3rdCheckAvailableViewDialogContent } from './-check-available';
export const Route = createFileRoute('/_app/credential3rd/manage')({
component: CredentialManageRouteComponent,
@@ -246,7 +250,18 @@ function CredentialManageRouteComponent() {
});
}}
onDelete={handleDeleteRecord(row)}
/>
>
<Dialog>
<DialogTrigger asChild>
<DropdownMenuItem onSelect={(e) => e.preventDefault()}>
Check Available
</DropdownMenuItem>
</DialogTrigger>
<Credential3rdCheckAvailableViewDialogContent
id={row.original.id}
/>
</Dialog>
</DataTableRowActions>
),
},
];

2
apps/webui/src/presentation/routes/_app/subscriptions/-sync.tsx
View File

@@ -126,7 +126,7 @@ export const SubscriptionSyncView = memo(
export interface SubscriptionSyncDialogContentProps {
id: number;
onCancel: VoidFunction;
onCancel?: VoidFunction;
}
export const SubscriptionSyncDialogContent = memo(

14
apps/webui/src/presentation/routes/_app/subscriptions/manage.tsx
View File

@@ -2,6 +2,8 @@ import { Button } from '@/components/ui/button';
import { DataTablePagination } from '@/components/ui/data-table-pagination';
import { DataTableRowActions } from '@/components/ui/data-table-row-actions';
import { DataTableViewOptions } from '@/components/ui/data-table-view-options';
import { Dialog, DialogTrigger } from '@/components/ui/dialog';
import { DropdownMenuItem } from '@/components/ui/dropdown-menu';
import { QueryErrorView } from '@/components/ui/query-error-view';
import { Skeleton } from '@/components/ui/skeleton';
import { Switch } from '@/components/ui/switch';
@@ -49,6 +51,7 @@ import { format } from 'date-fns';
import { Plus } from 'lucide-react';
import { useMemo, useState } from 'react';
import { toast } from 'sonner';
import { SubscriptionSyncDialogContent } from './-sync';
export const Route = createFileRoute('/_app/subscriptions/manage')({
component: SubscriptionManageRouteComponent,
@@ -240,7 +243,16 @@ function SubscriptionManageRouteComponent() {
});
}}
onDelete={handleDeleteRecord(row)}
/>
>
<Dialog>
<DialogTrigger asChild>
<DropdownMenuItem onSelect={(e) => e.preventDefault()}>
Sync
</DropdownMenuItem>
</DialogTrigger>
<SubscriptionSyncDialogContent id={row.original.id} />
</Dialog>
</DataTableRowActions>
),
},
];