dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add sync subscription webui and check credential web ui

Browse Source
This commit is contained in:
master
2025-06-08 00:36:59 +08:00
parent 946d4e8c2c
commit d2aab7369d
46 changed files with 1120 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
apps/recorder/.env
View File

@@ -11,3 +11,6 @@ BASIC_PASSWORD = "konobangu"
# OIDC_EXTRA_SCOPES = "read:konobangu write:konobangu"
# OIDC_EXTRA_CLAIM_KEY = ""
# OIDC_EXTRA_CLAIM_VALUE = ""
# MIKAN_PROXY = ""
# MIKAN_PROXY_AUTH_HEADER = ""
# MIKAN_NO_PROXY = ""

2
apps/recorder/examples/mikan_doppel_season_subscription.rs
View File

@@ -65,7 +65,7 @@ async fn main() -> Result<()> {
.prompt()?;
let mikan_scrape_client = mikan_scrape_client
.fork_with_credential(UserPassCredential {
.fork_with_userpass_credential(UserPassCredential {
username,
password,
user_agent: None,

8
apps/recorder/recorder.config.toml
View File

@@ -86,6 +86,14 @@ leaky_bucket_initial_tokens = 1
leaky_bucket_refill_tokens = 1
leaky_bucket_refill_interval = 500
[mikan.http_client.proxy]
server = '{{ get_env(name="MIKAN_PROXY", default = "") }}'
auth_header = '{{ get_env(name="MIKAN_PROXY_AUTH_HEADER", default = "") }}'
no_proxy = '{{ get_env(name="MIKAN_NO_PROXY", default = "") }}'
accept_invalid_certs = '{{ get_env(name="MIKAN_PROXY_ACCEPT_INVALID_CERTS", default = "false") }}'
[auth]
auth_type = '{{ get_env(name="AUTH_TYPE", default = "basic") }}'
basic_user = '{{ get_env(name="BASIC_USER", default = "konobangu") }}'

4
apps/recorder/src/app/config/default_mixin.toml
View File

@@ -11,6 +11,10 @@ leaky_bucket_initial_tokens = 0
leaky_bucket_refill_tokens = 1
leaky_bucket_refill_interval = 500
[mikan.http_client.proxy]
[mikan.http_client.proxy.headers]
[graphql]
depth_limit = inf
complexity_limit = inf

32
apps/recorder/src/extract/mikan/client.rs
View File

@@ -1,4 +1,4 @@
use std::{fmt::Debug, ops::Deref, sync::Arc};
use std::{fmt::Debug, ops::Deref};
use fetch::{HttpClient, HttpClientTrait};
use maplit::hashmap;
@@ -136,7 +136,7 @@ impl MikanClient {
pub async fn submit_credential_form(
&self,
ctx: Arc<dyn AppContextTrait>,
ctx: &dyn AppContextTrait,
subscriber_id: i32,
credential_form: MikanCredentialForm,
) -> RecorderResult<credential_3rd::Model> {
@@ -149,7 +149,7 @@ impl MikanClient {
subscriber_id: Set(subscriber_id),
..Default::default()
}
.try_encrypt(ctx.clone())
.try_encrypt(ctx)
.await?;
let credential: credential_3rd::Model = am.save(db).await?.try_into_model()?;
@@ -158,8 +158,9 @@ impl MikanClient {
pub async fn sync_credential_cookies(
&self,
ctx: Arc<dyn AppContextTrait>,
ctx: &dyn AppContextTrait,
credential_id: i32,
subscriber_id: i32,
) -> RecorderResult<()> {
let cookies = self.http_client.save_cookie_store_to_json()?;
if let Some(cookies) = cookies {
@@ -167,19 +168,20 @@ impl MikanClient {
cookies: Set(Some(cookies)),
..Default::default()
}
.try_encrypt(ctx.clone())
.try_encrypt(ctx)
.await?;
credential_3rd::Entity::update_many()
.set(am)
.filter(credential_3rd::Column::Id.eq(credential_id))
.filter(credential_3rd::Column::SubscriberId.eq(subscriber_id))
.exec(ctx.db())
.await?;
}
Ok(())
}
pub async fn fork_with_credential(
pub async fn fork_with_userpass_credential(
&self,
userpass_credential: UserPassCredential,
) -> RecorderResult<Self> {
@@ -204,10 +206,13 @@ impl MikanClient {
pub async fn fork_with_credential_id(
&self,
ctx: Arc<dyn AppContextTrait>,
ctx: &dyn AppContextTrait,
credential_id: i32,
subscriber_id: i32,
) -> RecorderResult<Self> {
let credential = credential_3rd::Model::find_by_id(ctx.clone(), credential_id).await?;
let credential =
credential_3rd::Model::find_by_id_and_subscriber_id(ctx, credential_id, subscriber_id)
.await?;
if let Some(credential) = credential {
if credential.credential_type != Credential3rdType::Mikan {
return Err(RecorderError::Credential3rdError {
@@ -219,7 +224,8 @@ impl MikanClient {
let userpass_credential: UserPassCredential =
credential.try_into_userpass_credential(ctx)?;
self.fork_with_credential(userpass_credential).await
self.fork_with_userpass_credential(userpass_credential)
.await
} else {
Err(RecorderError::from_db_record_not_found(
DbErr::RecordNotFound(format!("credential={credential_id} not found")),
@@ -249,7 +255,7 @@ impl HttpClientTrait for MikanClient {}
#[cfg(test)]
mod tests {
#![allow(unused_variables)]
use std::assert_matches::assert_matches;
use std::{assert_matches::assert_matches, sync::Arc};
use rstest::{fixture, rstest};
use tracing::Level;
@@ -297,8 +303,10 @@ mod tests {
let credential_form = build_testing_mikan_credential_form();
let subscriber_id = 1;
let credential_model = mikan_client
.submit_credential_form(app_ctx.clone(), 1, credential_form.clone())
.submit_credential_form(app_ctx.as_ref(), subscriber_id, credential_form.clone())
.await?;
let expected_username = &credential_form.username;
@@ -322,7 +330,7 @@ mod tests {
);
let mikan_client = mikan_client
.fork_with_credential_id(app_ctx.clone(), credential_model.id)
.fork_with_credential_id(app_ctx.as_ref(), credential_model.id, subscriber_id)
.await?;
mikan_client.login().await?;

2
apps/recorder/src/extract/mikan/config.rs
View File

@@ -2,7 +2,7 @@ use fetch::HttpClientConfig;
use serde::{Deserialize, Serialize};
use url::Url;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MikanConfig {
pub http_client: HttpClientConfig,
pub base_url: Url,

1
apps/recorder/src/extract/mikan/subscription.rs
View File

@@ -387,6 +387,7 @@ impl MikanSeasonSubscription {
ctx,
mikan_season_flow_url,
credential_id,
self.get_subscriber_id(),
)
}

20
apps/recorder/src/extract/mikan/web.rs
View File

@@ -917,10 +917,11 @@ pub fn scrape_mikan_bangumi_meta_stream_from_season_flow_url(
ctx: Arc<dyn AppContextTrait>,
mikan_season_flow_url: Url,
credential_id: i32,
subscriber_id: i32,
) -> impl Stream<Item = RecorderResult<MikanBangumiMeta>> {
try_stream! {
let mikan_base_url = ctx.mikan().base_url().clone();
let mikan_client = ctx.mikan().fork_with_credential_id(ctx.clone(), credential_id).await?;
let mikan_client = ctx.mikan().fork_with_credential_id(ctx.as_ref(), credential_id, subscriber_id).await?;
let content = fetch_html(&mikan_client, mikan_season_flow_url.clone()).await?;
@@ -940,7 +941,7 @@ pub fn scrape_mikan_bangumi_meta_stream_from_season_flow_url(
mikan_client
.sync_credential_cookies(ctx.clone(), credential_id)
.sync_credential_cookies(ctx.as_ref(), credential_id, subscriber_id)
.await?;
for bangumi_index in bangumi_indices_meta {
@@ -969,7 +970,7 @@ pub fn scrape_mikan_bangumi_meta_stream_from_season_flow_url(
}
mikan_client
.sync_credential_cookies(ctx, credential_id)
.sync_credential_cookies(ctx.as_ref(), credential_id, subscriber_id)
.await?;
}
}
@@ -978,11 +979,13 @@ pub async fn scrape_mikan_bangumi_meta_list_from_season_flow_url(
ctx: Arc<dyn AppContextTrait>,
mikan_season_flow_url: Url,
credential_id: i32,
subscriber_id: i32,
) -> RecorderResult<Vec<MikanBangumiMeta>> {
let stream = scrape_mikan_bangumi_meta_stream_from_season_flow_url(
ctx,
mikan_season_flow_url,
credential_id,
subscriber_id,
);
pin_mut!(stream);
@@ -1160,7 +1163,7 @@ mod test {
let mikan_client = build_testing_mikan_client(mikan_base_url.clone())
.await?
.fork_with_credential(build_testing_mikan_credential())
.fork_with_userpass_credential(build_testing_mikan_credential())
.await?;
mikan_client.login().await?;
@@ -1268,8 +1271,14 @@ mod test {
let mikan_client = app_ctx.mikan();
let subscriber_id = 1;
let credential = mikan_client
.submit_credential_form(app_ctx.clone(), 1, build_testing_mikan_credential_form())
.submit_credential_form(
app_ctx.as_ref(),
subscriber_id,
build_testing_mikan_credential_form(),
)
.await?;
let mikan_season_flow_url =
@@ -1279,6 +1288,7 @@ mod test {
app_ctx.clone(),
mikan_season_flow_url,
credential.id,
subscriber_id,
);
pin_mut!(bangumi_meta_stream);

6
apps/recorder/src/graphql/schema.rs
View File

@@ -20,7 +20,7 @@ use crate::{
},
util::{get_entity_column_key, get_entity_key},
},
views::register_subscriptions_to_schema,
views::{register_credential3rd_to_schema, register_subscriptions_to_schema},
},
};
@@ -156,7 +156,7 @@ pub fn build_schema(
&mut context,
&subscriber_tasks::Column::Job,
);
add_crypto_transformers(&mut context, app_ctx);
add_crypto_transformers(&mut context, app_ctx.clone());
for column in subscribers::Column::iter() {
if !matches!(column, subscribers::Column::Id) {
restrict_filter_input_for_entity::<subscribers::Entity>(
@@ -215,6 +215,7 @@ pub fn build_schema(
{
builder = register_subscriptions_to_schema(builder);
builder = register_credential3rd_to_schema(builder);
}
let schema = builder.schema_builder();
@@ -231,6 +232,7 @@ pub fn build_schema(
};
schema
.data(database)
.data(app_ctx)
.finish()
.inspect_err(|e| tracing::error!(e = ?e))
}

113
apps/recorder/src/graphql/views/credential_3rd.rs Normal file
View File

@@ -0,0 +1,113 @@
use std::sync::Arc;
use async_graphql::dynamic::{
Field, FieldFuture, FieldValue, InputObject, InputValue, Object, TypeRef,
};
use seaography::Builder as SeaographyBuilder;
use serde::{Deserialize, Serialize};
use util_derive::DynamicGraphql;
use crate::{
app::AppContextTrait, auth::AuthUserInfo, errors::RecorderError, models::credential_3rd,
};
#[derive(DynamicGraphql, Serialize, Deserialize, Clone, Debug)]
struct Credential3rdCheckAvailableInput {
pub id: i32,
}
impl Credential3rdCheckAvailableInput {
fn input_type_name() -> &'static str {
"Credential3rdCheckAvailableInput"
}
fn arg_name() -> &'static str {
"filter"
}
fn generate_input_object() -> InputObject {
InputObject::new(Self::input_type_name())
.description("The input of the credential3rdCheckAvailable query")
.field(InputValue::new(
Credential3rdCheckAvailableInputFieldEnum::Id.as_str(),
TypeRef::named_nn(TypeRef::INT),
))
}
}
#[derive(DynamicGraphql, Serialize, Deserialize, Clone, Debug)]
pub struct Credential3rdCheckAvailableInfo {
pub available: bool,
}
impl Credential3rdCheckAvailableInfo {
fn object_type_name() -> &'static str {
"Credential3rdCheckAvailableInfo"
}
fn generate_output_object() -> Object {
Object::new(Self::object_type_name())
.description("The output of the credential3rdCheckAvailable query")
.field(Field::new(
Credential3rdCheckAvailableInfoFieldEnum::Available,
TypeRef::named_nn(TypeRef::BOOLEAN),
move |ctx| {
FieldFuture::new(async move {
let subscription_info = ctx.parent_value.try_downcast_ref::<Self>()?;
Ok(Some(async_graphql::Value::from(
subscription_info.available,
)))
})
},
))
}
}
pub fn register_credential3rd_to_schema(mut builder: SeaographyBuilder) -> SeaographyBuilder {
builder.schema = builder
.schema
.register(Credential3rdCheckAvailableInput::generate_input_object());
builder.schema = builder
.schema
.register(Credential3rdCheckAvailableInfo::generate_output_object());
builder.queries.push(
Field::new(
"credential3rdCheckAvailable",
TypeRef::named_nn(Credential3rdCheckAvailableInfo::object_type_name()),
move |ctx| {
FieldFuture::new(async move {
let auth_user_info = ctx.data::<AuthUserInfo>()?;
let input: Credential3rdCheckAvailableInput = ctx
.args
.get(Credential3rdCheckAvailableInput::arg_name())
.unwrap()
.deserialize()?;
let app_ctx = ctx.data::<Arc<dyn AppContextTrait>>()?;
let credential_model = credential_3rd::Model::find_by_id_and_subscriber_id(
app_ctx.as_ref(),
input.id,
auth_user_info.subscriber_auth.subscriber_id,
)
.await?
.ok_or_else(|| RecorderError::Credential3rdError {
message: format!("credential = {} not found", input.id),
source: None.into(),
})?;
let available = credential_model.check_available(app_ctx.as_ref()).await?;
Ok(Some(FieldValue::owned_any(
Credential3rdCheckAvailableInfo { available },
)))
})
},
)
.argument(InputValue::new(
Credential3rdCheckAvailableInput::arg_name(),
TypeRef::named_nn(Credential3rdCheckAvailableInput::input_type_name()),
)),
);
builder
}

2
apps/recorder/src/graphql/views/mod.rs
View File

@@ -1,3 +1,5 @@
mod credential_3rd;
mod subscription;
pub use credential_3rd::register_credential3rd_to_schema;
pub use subscription::register_subscriptions_to_schema;

14
apps/recorder/src/graphql/views/subscription.rs
View File

@@ -16,7 +16,7 @@ use crate::{
#[derive(DynamicGraphql, Serialize, Deserialize, Clone, Debug)]
struct SyncOneSubscriptionFilterInput {
pub subscription_id: i32,
pub id: i32,
}
impl SyncOneSubscriptionFilterInput {
@@ -32,7 +32,7 @@ impl SyncOneSubscriptionFilterInput {
InputObject::new(Self::input_type_name())
.description("The input of the subscriptionSyncOne series of mutations")
.field(InputValue::new(
SyncOneSubscriptionFilterInputFieldEnum::SubscriptionId.as_str(),
SyncOneSubscriptionFilterInputFieldEnum::Id.as_str(),
TypeRef::named_nn(TypeRef::INT),
))
}
@@ -74,7 +74,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
.schema
.register(SyncOneSubscriptionInfo::generate_output_object());
builder.queries.push(
builder.mutations.push(
Field::new(
"subscriptionSyncOneFeedsIncremental",
TypeRef::named_nn(SyncOneSubscriptionInfo::object_type_name()),
@@ -93,7 +93,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let subscription_model = subscriptions::Model::find_by_id_and_subscriber_id(
app_ctx.as_ref(),
filter_input.subscription_id,
filter_input.id,
subscriber_id,
)
.await?;
@@ -124,7 +124,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
)),
);
builder.queries.push(
builder.mutations.push(
Field::new(
"subscriptionSyncOneFeedsFull",
TypeRef::named_nn(SyncOneSubscriptionInfo::object_type_name()),
@@ -143,7 +143,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let subscription_model = subscriptions::Model::find_by_id_and_subscriber_id(
app_ctx.as_ref(),
filter_input.subscription_id,
filter_input.id,
subscriber_id,
)
.await?;
@@ -193,7 +193,7 @@ pub fn register_subscriptions_to_schema(mut builder: SeaographyBuilder) -> Seaog
let subscription_model = subscriptions::Model::find_by_id_and_subscriber_id(
app_ctx.as_ref(),
filter_input.subscription_id,
filter_input.id,
subscriber_id,
)
.await?;

44
apps/recorder/src/models/credential_3rd.rs
View File

@@ -1,5 +1,3 @@
use std::sync::Arc;
use async_trait::async_trait;
use sea_orm::{ActiveValue, prelude::*};
use serde::{Deserialize, Serialize};
@@ -79,7 +77,7 @@ pub enum RelatedEntity {
impl ActiveModelBehavior for ActiveModel {}
impl ActiveModel {
pub async fn try_encrypt(mut self, ctx: Arc<dyn AppContextTrait>) -> RecorderResult<Self> {
pub async fn try_encrypt(mut self, ctx: &dyn AppContextTrait) -> RecorderResult<Self> {
let crypto = ctx.crypto();
if let ActiveValue::Set(Some(username)) = self.username {
@@ -102,19 +100,24 @@ impl ActiveModel {
}
impl Model {
pub async fn find_by_id(
ctx: Arc<dyn AppContextTrait>,
pub async fn find_by_id_and_subscriber_id(
ctx: &dyn AppContextTrait,
id: i32,
subscriber_id: i32,
) -> RecorderResult<Option<Self>> {
let db = ctx.db();
let credential = Entity::find_by_id(id).one(db).await?;
let credential = Entity::find()
.filter(Column::Id.eq(id))
.filter(Column::SubscriberId.eq(subscriber_id))
.one(db)
.await?;
Ok(credential)
}
pub fn try_into_userpass_credential(
self,
ctx: Arc<dyn AppContextTrait>,
ctx: &dyn AppContextTrait,
) -> RecorderResult<UserPassCredential> {
let crypto = ctx.crypto();
let username_enc = self
@@ -149,4 +152,31 @@ impl Model {
user_agent: self.user_agent,
})
}
pub async fn check_available(self, ctx: &dyn AppContextTrait) -> RecorderResult<bool> {
let credential_id = self.id;
let subscriber_id = self.subscriber_id;
match self.credential_type {
Credential3rdType::Mikan => {
let mikan_client = {
let userpass_credential: UserPassCredential =
self.try_into_userpass_credential(ctx)?;
ctx.mikan()
.fork_with_userpass_credential(userpass_credential)
.await?
};
let mut has_login = mikan_client.has_login().await?;
if !has_login {
mikan_client.login().await?;
has_login = true;
}
if has_login {
mikan_client
.sync_credential_cookies(ctx, credential_id, subscriber_id)
.await?;
}
Ok(has_login)
}
}
}
}