dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

temp save

Browse Source
This commit is contained in:
master 2025-07-04 05:59:56 +08:00
parent 147df00155
commit a1c2eeded1
17 changed files with 207 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@ -4846,15 +4846,6 @@ dependencies = [
"version_check",
]
[[package]]
name = "nanoid"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8"
dependencies = [
"rand 0.8.5",
]
[[package]]
name = "native-tls"
version = "0.2.14"
@ -6797,7 +6788,6 @@ dependencies = [
"mime_guess",
"mockito",
"moka",
"nanoid",
"nom 8.0.0",
"num-traits",
"num_cpus",
@ -6836,6 +6826,7 @@ dependencies = [
"tracing",
"tracing-appender",
"tracing-subscriber",
"tracing-test",
"tracing-tree",
"ts-rs",
"typed-builder 0.21.0",
@ -9244,6 +9235,27 @@ dependencies = [
"tracing-serde",
]
[[package]]
name = "tracing-test"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "557b891436fe0d5e0e363427fc7f217abf9ccd510d5136549847bdcbcd011d68"
dependencies = [
"tracing-core",
"tracing-subscriber",
"tracing-test-macro",
]
[[package]]
name = "tracing-test-macro"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568"
dependencies = [
"quote",
"syn 2.0.104",
]
[[package]]
name = "tracing-tree"
version = "0.4.0"

3
Cargo.toml
View File

@ -64,7 +64,7 @@ convert_case = "0.8"
color-eyre = "0.6.5"
inquire = "0.7.5"
image = "0.25.6"
uuid = { version = "1.6.0", features = ["v4"] }
uuid = { version = "1.6.0", features = ["v7"] }
maplit = "1.0.2"
once_cell = "1.20.2"
rand = "0.9.1"
@ -83,5 +83,6 @@ typed-builder = "0.21.0"
nanoid = "0.4.0"
webp = "0.3.0"
[patch.crates-io]
seaography = { git = "https://github.com/dumtruck/seaography.git", rev = "9f7fc7c" }

4
apps/recorder/Cargo.toml
View File

@ -97,7 +97,6 @@ tracing-appender = { workspace = true }
clap = { workspace = true }
ipnetwork = { workspace = true }
typed-builder = { workspace = true }
nanoid = { workspace = true }
webp = { workspace = true }
sea-orm = { version = "1.1", features = [
@ -176,5 +175,6 @@ inquire = { workspace = true }
color-eyre = { workspace = true }
serial_test = "3"
insta = { version = "1", features = ["redactions", "toml", "filters"] }
rstest = "0.25"
ctor = "0.4.0"
tracing-test = "0.2.5"
rstest = "0.25"

5
apps/recorder/src/graphql/domains/feeds.rs
View File

@ -3,6 +3,7 @@ use std::sync::Arc;
use async_graphql::dynamic::ResolverContext;
use sea_orm::Value as SeaValue;
use seaography::{Builder as SeaographyBuilder, BuilderContext, SeaResult};
use uuid::Uuid;
use crate::{
graphql::{
@ -35,7 +36,9 @@ pub fn register_feeds_to_schema_context(context: &mut BuilderContext) {
if field_name == entity_create_one_mutation_field_name.as_str()
|| field_name == entity_create_batch_mutation_field_name.as_str()
{
Ok(Some(SeaValue::String(Some(Box::new(nanoid::nanoid!())))))
Ok(Some(SeaValue::String(Some(Box::new(
Uuid::now_v7().to_string(),
)))))
} else {
Ok(None)
}

6
apps/recorder/src/migrations/m20250629_065628_add_cron.rs
View File

@ -139,7 +139,7 @@ impl MigrationTrait for Migration {
IF NEW.{next_run} IS NOT NULL
AND NEW.{next_run} <= CURRENT_TIMESTAMP
AND NEW.{enabled} = true
AND NEW.{status} = '{pending}'
AND NEW.{status} = '{pending}'::{status_type}
AND NEW.{attempts} < NEW.{max_attempts}
-- Check if not locked or lock timeout
AND (
@ -171,6 +171,7 @@ impl MigrationTrait for Migration {
pending = &CronStatus::Pending.to_value(),
attempts = &Cron::Attempts.to_string(),
max_attempts = &Cron::MaxAttempts.to_string(),
status_type = &CronStatus::name().to_string(),
))
.await?;
@ -194,7 +195,7 @@ impl MigrationTrait for Migration {
WHERE {next_run} IS NOT NULL
AND {next_run} <= CURRENT_TIMESTAMP
AND {enabled} = true
AND {status} = '{pending}'
AND {status} = '{pending}'::{status_type}
AND {attempts} < {max_attempts}
AND (
{locked_at} IS NULL
@ -222,6 +223,7 @@ impl MigrationTrait for Migration {
priority = &Cron::Priority.to_string(),
attempts = &Cron::Attempts.to_string(),
max_attempts = &Cron::MaxAttempts.to_string(),
status_type = &CronStatus::name().to_string(),
))
.await?;

2
apps/recorder/src/models/feeds/mod.rs
View File

@ -102,7 +102,7 @@ impl ActiveModelBehavior for ActiveModel {
C: ConnectionTrait,
{
if insert && let ActiveValue::NotSet = self.token {
let token = nanoid::nanoid!(10);
let token = Uuid::now_v7().to_string();
self.token = ActiveValue::Set(token);
}
Ok(self)

2
apps/recorder/src/storage/client.rs
View File

@ -278,7 +278,7 @@ impl StorageService {
if let Some(mut ranges) = ranges {
if ranges.len() > 1 {
let boundary = Uuid::new_v4().to_string();
let boundary = Uuid::now_v7().to_string();
let reader = self.reader(storage_path.as_ref()).await?;
let stream: impl Stream<Item = Result<Bytes, RecorderError>> = {
let boundary = boundary.clone();

7
apps/recorder/src/task/config.rs
View File

@ -14,6 +14,8 @@ pub struct TaskConfig {
pub system_task_reenqueue_orphaned_after: Duration,
#[serde(default = "default_cron_retry_duration")]
pub cron_retry_duration: Duration,
#[serde(default = "default_cron_interval_duration")]
pub cron_interval_duration: Duration,
}
impl Default for TaskConfig {
@ -25,6 +27,7 @@ impl Default for TaskConfig {
default_subscriber_task_reenqueue_orphaned_after(),
system_task_reenqueue_orphaned_after: default_system_task_reenqueue_orphaned_after(),
cron_retry_duration: default_cron_retry_duration(),
cron_interval_duration: default_cron_interval_duration(),
}
}
}
@ -45,6 +48,10 @@ pub fn default_system_task_workers() -> u32 {
}
}
pub fn default_cron_interval_duration() -> Duration {
Duration::from_secs(60)
}
pub fn default_subscriber_task_reenqueue_orphaned_after() -> Duration {
Duration::from_secs(3600)
}

2
apps/recorder/src/task/mod.rs
View File

@ -11,7 +11,7 @@ pub use core::{
pub use config::TaskConfig;
pub use registry::{
OptimizeImageTask, SubscriberTask, SubscriberTaskInput, SubscriberTaskType,
EchoTask, OptimizeImageTask, SubscriberTask, SubscriberTaskInput, SubscriberTaskType,
SubscriberTaskTypeEnum, SubscriberTaskTypeVariant, SubscriberTaskTypeVariantIter,
SyncOneSubscriptionFeedsFullTask, SyncOneSubscriptionFeedsIncrementalTask,
SyncOneSubscriptionSourcesTask, SystemTask, SystemTaskInput, SystemTaskType,

2
apps/recorder/src/task/registry/mod.rs
View File

@ -9,6 +9,6 @@ pub use subscriber::{
};
pub(crate) use system::register_system_task_type;
pub use system::{
OptimizeImageTask, SystemTask, SystemTaskInput, SystemTaskType, SystemTaskTypeEnum,
EchoTask, OptimizeImageTask, SystemTask, SystemTaskInput, SystemTaskType, SystemTaskTypeEnum,
SystemTaskTypeVariant, SystemTaskTypeVariantIter,
};

29
apps/recorder/src/task/registry/system/misc.rs Normal file
View File

@ -0,0 +1,29 @@
use std::sync::Arc;
use chrono::Utc;
use crate::{
app::AppContextTrait,
errors::RecorderResult,
task::{AsyncTaskTrait, register_system_task_type},
};
register_system_task_type! {
#[derive(Debug, Clone, PartialEq)]
pub struct EchoTask {
pub task_id: String,
}
}
#[async_trait::async_trait]
impl AsyncTaskTrait for EchoTask {
async fn run_async(self, _ctx: Arc<dyn AppContextTrait>) -> RecorderResult<()> {
tracing::info!(
"EchoTask {} start running at {}",
self.task_id,
Utc::now().to_rfc3339()
);
Ok(())
}
}

28
apps/recorder/src/task/registry/system/mod.rs
View File

@ -1,8 +1,10 @@
mod base;
mod media;
mod misc;
pub(crate) use base::register_system_task_type;
pub use media::OptimizeImageTask;
pub use misc::EchoTask;
use sea_orm::{DeriveActiveEnum, DeriveDisplay, EnumIter, FromJsonQueryResult};
macro_rules! register_system_task_types {
@ -131,30 +133,6 @@ macro_rules! register_system_task_types {
};
}
#[cfg(not(any(test, feature = "test-utils")))]
register_system_task_types! {
task_type_enum: {
#[derive(
Clone,
Debug,
Copy,
DeriveActiveEnum,
DeriveDisplay,
EnumIter,
)]
pub enum SystemTaskType {
OptimizeImage => "optimize_image"
}
},
task_enum: {
#[derive(Clone, Debug, FromJsonQueryResult)]
pub enum SystemTask {
OptimizeImage(OptimizeImageTask)
}
}
}
#[cfg(any(test, feature = "test-utils"))]
register_system_task_types! {
task_type_enum: {
#[derive(
@ -174,7 +152,7 @@ register_system_task_types! {
#[derive(Clone, Debug, FromJsonQueryResult)]
pub enum SystemTask {
OptimizeImage(OptimizeImageTask),
Test(crate::test_utils::task::TestSystemTask),
Echo(EchoTask),
}
}
}

99
apps/recorder/src/task/service.rs
View File

@ -6,8 +6,9 @@ use apalis_sql::{
context::SqlContext,
postgres::{PgListen as ApalisPgListen, PostgresStorage as ApalisPostgresStorage},
};
use sea_orm::sqlx::postgres::PgListener;
use sea_orm::{ActiveModelTrait, sqlx::postgres::PgListener};
use tokio::sync::RwLock;
use uuid::Uuid;
use crate::{
app::AppContextTrait,
@ -53,7 +54,7 @@ impl TaskService {
Ok(Self {
config,
cron_worker_id: nanoid::nanoid!(),
cron_worker_id: Uuid::now_v7().to_string(),
ctx,
subscriber_task_storage: Arc::new(RwLock::new(subscriber_task_storage)),
system_task_storage: Arc::new(RwLock::new(system_task_storage)),
@ -136,6 +137,21 @@ impl TaskService {
Ok(task_id)
}
pub async fn add_subscriber_task_cron(
&self,
cm: cron::ActiveModel,
) -> RecorderResult<cron::Model> {
let db = self.ctx.db();
let m = cm.insert(db).await?;
Ok(m)
}
pub async fn add_system_task_cron(&self, cm: cron::ActiveModel) -> RecorderResult<cron::Model> {
let db = self.ctx.db();
let m = cm.insert(db).await?;
Ok(m)
}
pub async fn run<F, Fut>(&self, shutdown_signal: Option<F>) -> RecorderResult<()>
where
F: Fn() -> Fut + Send + 'static,
@ -167,30 +183,31 @@ impl TaskService {
Ok::<_, RecorderError>(())
},
async {
let listener = self.setup_cron_due_listening().await?;
let ctx = self.ctx.clone();
let mut listener = self.setup_cron_due_listening().await?;
let cron_worker_id = self.cron_worker_id.clone();
let retry_duration = chrono::Duration::milliseconds(
self.config.cron_retry_duration.as_millis() as i64,
);
let cron_interval_duration = self.config.cron_interval_duration;
listener.listen(CRON_DUE_EVENT).await?;
tracing::debug!("Listening for cron due event...");
tokio::task::spawn(async move {
tokio::task::spawn({
let ctx = self.ctx.clone();
async move {
if let Err(e) =
Self::listen_cron_due(listener, ctx, &cron_worker_id, retry_duration).await
Self::listen_cron_due(listener, ctx, &cron_worker_id, retry_duration)
.await
{
tracing::error!("Error listening to cron due: {e}");
}
}
});
Ok::<_, RecorderError>(())
},
async {
tokio::task::spawn({
let ctx = self.ctx.clone();
let retry_duration = chrono::Duration::milliseconds(
self.config.cron_retry_duration.as_millis() as i64,
);
tokio::task::spawn(async move {
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(60));
async move {
let mut interval = tokio::time::interval(cron_interval_duration);
loop {
interval.tick().await;
if let Err(e) = cron::Model::check_and_cleanup_expired_cron_locks(
@ -203,11 +220,13 @@ impl TaskService {
"Error checking and cleaning up expired cron locks: {e}"
);
}
if let Err(e) = cron::Model::check_and_trigger_due_crons(ctx.as_ref()).await
if let Err(e) =
cron::Model::check_and_trigger_due_crons(ctx.as_ref()).await
{
tracing::error!("Error checking and triggering due crons: {e}");
}
}
}
});
Ok::<_, RecorderError>(())
@ -267,6 +286,7 @@ impl TaskService {
async fn setup_cron_due_listening(&self) -> RecorderResult<PgListener> {
let pool = self.ctx.db().get_postgres_connection_pool().clone();
let listener = PgListener::connect_with(&pool).await?;
tracing::debug!("Cron due listener connected to postgres");
Ok(listener)
}
@ -277,10 +297,9 @@ impl TaskService {
worker_id: &str,
retry_duration: chrono::Duration,
) -> RecorderResult<()> {
listener.listen(CRON_DUE_EVENT).await?;
loop {
let notification = listener.recv().await?;
tracing::debug!("Received cron due event: {:?}", notification);
if let Err(e) = cron::Model::handle_cron_notification(
ctx.as_ref(),
notification,
@ -298,13 +317,20 @@ impl TaskService {
#[cfg(test)]
#[allow(unused_variables)]
mod tests {
use std::time::Duration;
use rstest::{fixture, rstest};
use sea_orm::ActiveValue;
use tracing::Level;
use super::*;
use crate::test_utils::{
// app::TestingPreset,
use crate::{
models::cron,
task::EchoTask,
test_utils::{
app::{TestingAppContextConfig, TestingPreset},
tracing::try_init_testing_tracing,
},
};
#[fixture]
@ -314,7 +340,40 @@ mod tests {
#[rstest]
#[tokio::test]
// #[tracing_test::traced_test]
async fn test_cron_due_listening(before_each: ()) -> RecorderResult<()> {
todo!()
let preset = TestingPreset::default_with_config(
TestingAppContextConfig::builder()
.task_config(TaskConfig {
cron_interval_duration: Duration::from_secs(1),
..Default::default()
})
.build(),
)
.await?;
let app_ctx = preset.app_ctx;
let task_service = app_ctx.task();
let task_id = Uuid::now_v7().to_string();
let echo_cron = cron::ActiveModel {
cron_expr: ActiveValue::Set("*/1 * * * * *".to_string()),
system_task_cron: ActiveValue::Set(Some(
EchoTask::builder().task_id(task_id.clone()).build().into(),
)),
..Default::default()
};
let _ = task_service
.run(Some(async move || {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}))
.await;
// assert!(logs_contain(&format!(
// "EchoTask {task_id} start running at"
// )));
Ok(())
}
}

38
apps/recorder/src/test_utils/app.rs
View File

@ -6,6 +6,7 @@ use typed_builder::TypedBuilder;
use crate::{
app::AppContextTrait,
errors::RecorderResult,
task::TaskConfig,
test_utils::{
crypto::build_testing_crypto_service,
database::{TestingDatabaseServiceConfig, build_testing_database_service},
@ -43,10 +44,11 @@ impl TestingAppContext {
self.task.get_or_init(|| task);
}
pub async fn from_preset(preset: TestingAppContextPreset) -> RecorderResult<Arc<Self>> {
let mikan_client = build_testing_mikan_client(preset.mikan_base_url).await?;
pub async fn from_config(config: TestingAppContextConfig) -> RecorderResult<Arc<Self>> {
let mikan_base_url = config.mikan_base_url.expect("mikan_base_url is required");
let mikan_client = build_testing_mikan_client(mikan_base_url).await?;
let db_service =
build_testing_database_service(preset.database_config.unwrap_or_default()).await?;
build_testing_database_service(config.database_config.unwrap_or_default()).await?;
let crypto_service = build_testing_crypto_service().await?;
let storage_service = build_testing_storage_service().await?;
let media_service = build_testing_media_service().await?;
@ -132,9 +134,12 @@ impl AppContextTrait for TestingAppContext {
}
}
pub struct TestingAppContextPreset {
pub mikan_base_url: String,
#[derive(TypedBuilder)]
#[builder(field_defaults(default, setter(strip_option)))]
pub struct TestingAppContextConfig {
pub mikan_base_url: Option<String>,
pub database_config: Option<TestingDatabaseServiceConfig>,
pub task_config: Option<TaskConfig>,
}
#[derive(TypedBuilder)]
@ -144,15 +149,15 @@ pub struct TestingPreset {
}
impl TestingPreset {
pub async fn default() -> RecorderResult<Self> {
pub async fn default_with_config(config: TestingAppContextConfig) -> RecorderResult<Self> {
let mikan_server = MikanMockServer::new().await?;
let database_config = TestingDatabaseServiceConfig::default();
let app_ctx = TestingAppContext::from_preset(TestingAppContextPreset {
mikan_base_url: mikan_server.base_url().to_string(),
database_config: Some(database_config),
})
.await?;
let mixed_config = TestingAppContextConfig {
mikan_base_url: Some(mikan_server.base_url().to_string()),
..config
};
let app_ctx = TestingAppContext::from_config(mixed_config).await?;
let preset = Self::builder()
.mikan_server(mikan_server)
@ -160,4 +165,13 @@ impl TestingPreset {
.build();
Ok(preset)
}
pub async fn default() -> RecorderResult<Self> {
Self::default_with_config(TestingAppContextConfig {
mikan_base_url: None,
database_config: None,
task_config: None,
})
.await
}
}

2
apps/recorder/src/test_utils/database.rs
View File

@ -52,7 +52,7 @@ pub async fn build_testing_database_service(
uri: connection_string,
enable_logging: true,
min_connections: 1,
max_connections: 1,
max_connections: 5,
connect_timeout: 5000,
idle_timeout: 10000,
acquire_timeout: None,

33
apps/recorder/src/test_utils/task.rs
View File

@ -1,42 +1,17 @@
use std::sync::Arc;
use chrono::Utc;
use crate::{
app::AppContextTrait,
errors::RecorderResult,
task::{AsyncTaskTrait, TaskConfig, TaskService, register_system_task_type},
task::{TaskConfig, TaskService},
};
register_system_task_type! {
#[derive(Debug, Clone, PartialEq)]
pub struct TestSystemTask {
pub task_id: String,
}
}
#[async_trait::async_trait]
impl AsyncTaskTrait for TestSystemTask {
async fn run_async(self, ctx: Arc<dyn AppContextTrait>) -> RecorderResult<()> {
let storage = ctx.storage();
storage
.write(
storage.build_test_path(self.task_id),
serde_json::json!({ "exec_time": Utc::now().timestamp_millis() })
.to_string()
.into(),
)
.await?;
Ok(())
}
}
pub async fn build_testing_task_service(
ctx: Arc<dyn AppContextTrait>,
) -> RecorderResult<TaskService> {
let config = TaskConfig::default();
let config = TaskConfig {
..Default::default()
};
let task_service = TaskService::from_config_and_ctx(config, ctx).await?;
Ok(task_service)

2
apps/recorder/src/web/middleware/request_id.rs
View File

@ -110,7 +110,7 @@ fn make_request_id(maybe_request_id: Option<HeaderValue>) -> String {
});
id.filter(|s| !s.is_empty())
})
.unwrap_or_else(|| Uuid::new_v4().to_string())
.unwrap_or_else(|| Uuid::now_v7().to_string())
}
#[cfg(test)]