dumtruck/konobangu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: init cron webui

Browse Source
This commit is contained in:
master
2025-07-05 02:08:55 +08:00
parent a1c2eeded1
commit 004fed9b2e
33 changed files with 3329 additions and 551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/recorder/src/app/core.rs
View File

@@ -107,7 +107,7 @@ impl App {
Ok::<(), RecorderError>(())
},
async {
task.run(if graceful_shutdown {
task.run_with_signal(if graceful_shutdown {
Some(Self::shutdown_signal)
} else {
None

25
apps/recorder/src/migrations/m20250629_065628_add_cron.rs
View File

@@ -8,9 +8,10 @@ use crate::{
Subscriptions, table_auto_z,
},
models::cron::{
CHECK_AND_TRIGGER_DUE_CRONS_FUNCTION_NAME, CRON_DUE_EVENT, CronStatus, CronStatusEnum,
NOTIFY_DUE_CRON_WHEN_MUTATING_FUNCTION_NAME, NOTIFY_DUE_CRON_WHEN_MUTATING_TRIGGER_NAME,
SETUP_CRON_EXTRA_FOREIGN_KEYS_FUNCTION_NAME, SETUP_CRON_EXTRA_FOREIGN_KEYS_TRIGGER_NAME,
CHECK_AND_TRIGGER_DUE_CRONS_FUNCTION_NAME, CRON_DUE_DEBUG_EVENT, CRON_DUE_EVENT,
CronStatus, CronStatusEnum, NOTIFY_DUE_CRON_WHEN_MUTATING_FUNCTION_NAME,
NOTIFY_DUE_CRON_WHEN_MUTATING_TRIGGER_NAME, SETUP_CRON_EXTRA_FOREIGN_KEYS_FUNCTION_NAME,
SETUP_CRON_EXTRA_FOREIGN_KEYS_TRIGGER_NAME,
},
task::{
SETUP_APALIS_JOBS_EXTRA_FOREIGN_KEYS_FUNCTION_NAME, SUBSCRIBER_TASK_APALIS_NAME,
@@ -48,14 +49,13 @@ impl MigrationTrait for Migration {
.col(string_null(Cron::LockedBy))
.col(timestamp_with_time_zone_null(Cron::LockedAt))
.col(integer_null(Cron::TimeoutMs))
.col(integer(Cron::Attempts))
.col(integer(Cron::MaxAttempts))
.col(integer(Cron::Priority))
.col(enumeration(
Cron::Status,
CronStatusEnum,
CronStatus::iden_values(),
))
.col(integer(Cron::Attempts).default(0))
.col(integer(Cron::MaxAttempts).default(1))
.col(integer(Cron::Priority).default(0))
.col(
enumeration(Cron::Status, CronStatusEnum, CronStatus::iden_values())
.default(CronStatus::Pending),
)
.col(json_binary_null(Cron::SubscriberTaskCron))
.col(json_binary_null(Cron::SystemTaskCron))
.foreign_key(
@@ -207,9 +207,12 @@ impl MigrationTrait for Migration {
ORDER BY {priority} ASC, {next_run} ASC
FOR UPDATE SKIP LOCKED
LOOP
-- PERFORM pg_notify('{CRON_DUE_DEBUG_EVENT}',format('Found due cron: value=%s; Now time: %s', row_to_json(cron_record)::text, CURRENT_TIMESTAMP));
PERFORM pg_notify('{CRON_DUE_EVENT}', row_to_json(cron_record)::text);
notification_count := notification_count + 1;
END LOOP;
-- PERFORM pg_notify('{CRON_DUE_DEBUG_EVENT}', format('Notification count: %I; Now time: %s', notification_count, CURRENT_TIMESTAMP));
RETURN notification_count;
END;
$$ LANGUAGE plpgsql;"#,

1
apps/recorder/src/models/cron/core.rs
View File

@@ -1,4 +1,5 @@
pub const CRON_DUE_EVENT: &str = "cron_due";
pub const CRON_DUE_DEBUG_EVENT: &str = "cron_due_debug";
pub const CHECK_AND_TRIGGER_DUE_CRONS_FUNCTION_NAME: &str = "check_and_trigger_due_crons";

23
apps/recorder/src/models/cron/mod.rs
View File

@@ -1,7 +1,7 @@
mod core;
pub use core::{
CHECK_AND_TRIGGER_DUE_CRONS_FUNCTION_NAME, CRON_DUE_EVENT,
CHECK_AND_TRIGGER_DUE_CRONS_FUNCTION_NAME, CRON_DUE_DEBUG_EVENT, CRON_DUE_EVENT,
NOTIFY_DUE_CRON_WHEN_MUTATING_FUNCTION_NAME, NOTIFY_DUE_CRON_WHEN_MUTATING_TRIGGER_NAME,
SETUP_CRON_EXTRA_FOREIGN_KEYS_FUNCTION_NAME, SETUP_CRON_EXTRA_FOREIGN_KEYS_TRIGGER_NAME,
};
@@ -59,8 +59,8 @@ pub struct Model {
pub last_error: Option<String>,
pub locked_by: Option<String>,
pub locked_at: Option<DateTimeUtc>,
#[sea_orm(default_expr = "5000")]
pub timeout_ms: i32,
// default_expr = "5000"
pub timeout_ms: Option<i32>,
#[sea_orm(default_expr = "0")]
pub attempts: i32,
#[sea_orm(default_expr = "1")]
@@ -223,7 +223,10 @@ impl Model {
&& cron.attempts < cron.max_attempts
&& cron.status == CronStatus::Pending
&& (cron.locked_at.is_none_or(|locked_at| {
locked_at + chrono::Duration::milliseconds(cron.timeout_ms as i64) <= Utc::now()
cron.timeout_ms.is_some_and(|cron_timeout_ms| {
locked_at + chrono::Duration::milliseconds(cron_timeout_ms as i64)
<= Utc::now()
})
}))
&& cron.next_run.is_some_and(|next_run| next_run <= Utc::now())
{
@@ -376,7 +379,15 @@ impl Model {
locked_cron
.mark_cron_failed(
ctx,
format!("Cron timeout of {}ms", locked_cron.timeout_ms).as_str(),
format!(
"Cron timeout of {}ms",
locked_cron
.timeout_ms
.as_ref()
.map(|s| s.to_string())
.unwrap_or_else(|| "Infinite".to_string())
)
.as_str(),
retry_duration,
)
.await?;
@@ -389,7 +400,7 @@ impl Model {
}
pub fn calculate_next_run(cron_expr: &str) -> RecorderResult<DateTime<Utc>> {
let cron_expr = Cron::new(cron_expr).parse()?;
let cron_expr = Cron::new(cron_expr).with_seconds_optional().parse()?;
let next = cron_expr.find_next_occurrence(&Utc::now(), false)?;

10
apps/recorder/src/models/subscriptions/mod.rs
View File

@@ -60,6 +60,8 @@ pub enum Relation {
Feed,
#[sea_orm(has_many = "super::subscriber_tasks::Entity")]
SubscriberTask,
#[sea_orm(has_many = "super::cron::Entity")]
Cron,
}
impl Related<super::subscribers::Entity> for Entity {
@@ -126,6 +128,12 @@ impl Related<super::subscriber_tasks::Entity> for Entity {
}
}
impl Related<super::cron::Entity> for Entity {
fn to() -> RelationDef {
Relation::Cron.def()
}
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelatedEntity)]
pub enum RelatedEntity {
#[sea_orm(entity = "super::subscribers::Entity")]
@@ -144,6 +152,8 @@ pub enum RelatedEntity {
Feed,
#[sea_orm(entity = "super::subscriber_tasks::Entity")]
SubscriberTask,
#[sea_orm(entity = "super::cron::Entity")]
Cron,
}
#[async_trait]

217
apps/recorder/src/task/service.rs
View File

@@ -13,7 +13,7 @@ use uuid::Uuid;
use crate::{
app::AppContextTrait,
errors::{RecorderError, RecorderResult},
models::cron::{self, CRON_DUE_EVENT},
models::cron::{self, CRON_DUE_DEBUG_EVENT, CRON_DUE_EVENT},
task::{
AsyncTaskTrait, SUBSCRIBER_TASK_APALIS_NAME, SYSTEM_TASK_APALIS_NAME, SubscriberTask,
TaskConfig,
@@ -152,86 +152,95 @@ impl TaskService {
Ok(m)
}
pub async fn run<F, Fut>(&self, shutdown_signal: Option<F>) -> RecorderResult<()>
pub async fn run(&self) -> RecorderResult<()> {
self.run_with_signal(None::<fn() -> std::future::Ready<()>>)
.await
}
pub async fn run_with_signal<F, Fut>(&self, shutdown_signal: Option<F>) -> RecorderResult<()>
where
F: Fn() -> Fut + Send + 'static,
F: FnOnce() -> Fut + Send + 'static,
Fut: Future<Output = ()> + Send,
{
tokio::try_join!(
async {
tokio::select! {
_ = {
let monitor = self.setup_apalis_monitor().await?;
if let Some(shutdown_signal) = shutdown_signal {
monitor
.run_with_signal(async move {
shutdown_signal().await;
tracing::info!("apalis shutting down...");
Ok(())
})
.await?;
} else {
monitor.run().await?;
async move {
if let Some(shutdown_signal) = shutdown_signal {
monitor
.run_with_signal(async move {
shutdown_signal().await;
tracing::info!("apalis shutting down...");
Ok(())
})
.await?;
} else {
monitor.run().await?;
}
Ok::<_, RecorderError>(())
}
Ok::<_, RecorderError>(())
},
async {
} => {}
_ = {
let listener = self.setup_apalis_listener().await?;
tokio::task::spawn(async move {
async move {
if let Err(e) = listener.listen().await {
tracing::error!("Error listening to apalis: {e}");
}
});
Ok::<_, RecorderError>(())
},
async {
Ok::<_, RecorderError>(())
}
} => {},
_ = {
let mut listener = self.setup_cron_due_listening().await?;
let cron_worker_id = self.cron_worker_id.clone();
let retry_duration = chrono::Duration::milliseconds(
self.config.cron_retry_duration.as_millis() as i64,
);
let retry_duration =
chrono::Duration::milliseconds(self.config.cron_retry_duration.as_millis() as i64);
let cron_interval_duration = self.config.cron_interval_duration;
listener.listen(CRON_DUE_EVENT).await?;
tracing::debug!("Listening for cron due event...");
async move {
listener.listen_all([CRON_DUE_EVENT as &str, CRON_DUE_DEBUG_EVENT as &str]).await?;
tokio::task::spawn({
let ctx = self.ctx.clone();
async move {
if let Err(e) =
Self::listen_cron_due(listener, ctx, &cron_worker_id, retry_duration)
.await
tokio::join!(
{
tracing::error!("Error listening to cron due: {e}");
}
}
});
tokio::task::spawn({
let ctx = self.ctx.clone();
async move {
let mut interval = tokio::time::interval(cron_interval_duration);
loop {
interval.tick().await;
if let Err(e) = cron::Model::check_and_cleanup_expired_cron_locks(
ctx.as_ref(),
retry_duration,
)
.await
{
tracing::error!(
"Error checking and cleaning up expired cron locks: {e}"
);
let ctx = self.ctx.clone();
async move {
if let Err(e) =
Self::listen_cron_due(listener, ctx, &cron_worker_id, retry_duration)
.await
{
tracing::error!("Error listening to cron due: {e}");
}
}
if let Err(e) =
cron::Model::check_and_trigger_due_crons(ctx.as_ref()).await
{
tracing::error!("Error checking and triggering due crons: {e}");
},
{
let ctx = self.ctx.clone();
let mut interval = tokio::time::interval(cron_interval_duration);
async move {
loop {
interval.tick().await;
if let Err(e) = cron::Model::check_and_cleanup_expired_cron_locks(
ctx.as_ref(),
retry_duration,
)
.await
{
tracing::error!(
"Error checking and cleaning up expired cron locks: {e}"
);
}
if let Err(e) =
cron::Model::check_and_trigger_due_crons(ctx.as_ref()).await
{
tracing::error!("Error checking and triggering due crons: {e}");
}
}
}
}
}
});
);
Ok::<_, RecorderError>(())
}
} => {}
};
Ok::<_, RecorderError>(())
}
)?;
Ok(())
}
@@ -299,14 +308,17 @@ impl TaskService {
) -> RecorderResult<()> {
loop {
let notification = listener.recv().await?;
tracing::debug!("Received cron due event: {:?}", notification);
if let Err(e) = cron::Model::handle_cron_notification(
ctx.as_ref(),
notification,
worker_id,
retry_duration,
)
.await
if notification.channel() == CRON_DUE_DEBUG_EVENT {
tracing::debug!("Received cron due debug event: {:?}", notification);
continue;
} else if notification.channel() == CRON_DUE_EVENT
&& let Err(e) = cron::Model::handle_cron_notification(
ctx.as_ref(),
notification,
worker_id,
retry_duration,
)
.await
{
tracing::error!("Error handling cron notification: {e}");
}
@@ -319,6 +331,7 @@ impl TaskService {
mod tests {
use std::time::Duration;
use chrono::Utc;
use rstest::{fixture, rstest};
use sea_orm::ActiveValue;
use tracing::Level;
@@ -340,12 +353,14 @@ mod tests {
#[rstest]
#[tokio::test]
// #[tracing_test::traced_test]
async fn test_cron_due_listening(before_each: ()) -> RecorderResult<()> {
#[tracing_test::traced_test]
async fn test_check_and_trigger_due_crons_with_certain_interval(
before_each: (),
) -> RecorderResult<()> {
let preset = TestingPreset::default_with_config(
TestingAppContextConfig::builder()
.task_config(TaskConfig {
cron_interval_duration: Duration::from_secs(1),
cron_interval_duration: Duration::from_millis(1500),
..Default::default()
})
.build(),
@@ -364,15 +379,53 @@ mod tests {
..Default::default()
};
let _ = task_service
.run(Some(async move || {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}))
.await;
task_service.add_system_task_cron(echo_cron).await?;
// assert!(logs_contain(&format!(
// "EchoTask {task_id} start running at"
// )));
task_service
.run_with_signal(Some(async move || {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}))
.await?;
assert!(logs_contain(&format!(
"EchoTask {task_id} start running at"
)));
Ok(())
}
#[rstest]
#[tokio::test]
#[tracing_test::traced_test]
async fn test_trigger_due_cron_when_mutating(before_each: ()) -> RecorderResult<()> {
let preset = TestingPreset::default().await?;
let app_ctx = preset.app_ctx;
let task_service = app_ctx.task();
let task_id = Uuid::now_v7().to_string();
let echo_cron = cron::ActiveModel {
cron_expr: ActiveValue::Set("* * * */1 * *".to_string()),
next_run: ActiveValue::Set(Some(Utc::now() + chrono::Duration::seconds(-10))),
system_task_cron: ActiveValue::Set(Some(
EchoTask::builder().task_id(task_id.clone()).build().into(),
)),
..Default::default()
};
let task_runner = task_service.run_with_signal(Some(async move || {
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}));
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
task_service.add_system_task_cron(echo_cron).await?;
task_runner.await?;
assert!(logs_contain(&format!(
"EchoTask {task_id} start running at"
)));
Ok(())
}

4
apps/recorder/src/test_utils/app.rs
View File

@@ -62,7 +62,7 @@ impl TestingAppContext {
.build(),
);
let task_service = build_testing_task_service(app_ctx.clone()).await?;
let task_service = build_testing_task_service(config.task_config, app_ctx.clone()).await?;
app_ctx.set_task(task_service);
@@ -134,7 +134,7 @@ impl AppContextTrait for TestingAppContext {
}
}
#[derive(TypedBuilder)]
#[derive(TypedBuilder, Debug)]
#[builder(field_defaults(default, setter(strip_option)))]
pub struct TestingAppContextConfig {
pub mikan_base_url: Option<String>,

5
apps/recorder/src/test_utils/task.rs
View File

@@ -7,11 +7,10 @@ use crate::{
};
pub async fn build_testing_task_service(
config: Option<TaskConfig>,
ctx: Arc<dyn AppContextTrait>,
) -> RecorderResult<TaskService> {
let config = TaskConfig {
..Default::default()
};
let config = config.unwrap_or_default();
let task_service = TaskService::from_config_and_ctx(config, ctx).await?;
Ok(task_service)