feat: 加固运行期可靠性并补齐调度与观测接口
- 运行句柄按 run_id 归属,修复取消后重跑时闸门被旧运行删除的竞态 - 调度循环改用取消令牌退出,启动时对账遗留运行,校验 rows_affected - 三类任务统一参与调度,新增 /api/v1/stats 与 /api/v1/schedule - SQLite 启用 WAL 与 busy_timeout,日志清理跳过活跃文件,单行上限 256 KiB - 前端缓存管理令牌并在 403 时重试一次,仪表盘改用统计接口
This commit is contained in:
@@ -17,7 +17,9 @@
|
|||||||
/data/
|
/data/
|
||||||
/logs/
|
/logs/
|
||||||
/.e2e-data/
|
/.e2e-data/
|
||||||
|
/.tmp/
|
||||||
/release/
|
/release/
|
||||||
|
/samples/python-task/.venv/
|
||||||
|
|
||||||
# Editors and operating system files
|
# Editors and operating system files
|
||||||
/.idea/
|
/.idea/
|
||||||
|
|||||||
+53
-3
@@ -1,6 +1,6 @@
|
|||||||
use axum::{
|
use axum::{
|
||||||
Json, Router,
|
Json, Router,
|
||||||
extract::State,
|
extract::{DefaultBodyLimit, State},
|
||||||
http::{HeaderValue, Method, header},
|
http::{HeaderValue, Method, header},
|
||||||
middleware,
|
middleware,
|
||||||
response::IntoResponse,
|
response::IntoResponse,
|
||||||
@@ -31,8 +31,9 @@ use crate::{
|
|||||||
state::AppState,
|
state::AppState,
|
||||||
tasks::{
|
tasks::{
|
||||||
model::{
|
model::{
|
||||||
ConcurrencyPolicy, CreateTaskRequest, HttpMethod, RetryPolicy, TaskAction,
|
ConcurrencyPolicy, CreateTaskRequest, HttpMethod, RetryPolicy, ScheduledTaskResponse,
|
||||||
TaskActionType, TaskListResponse, TaskResponse, UpdateTaskRequest,
|
TaskAction, TaskActionType, TaskListResponse, TaskResponse, TaskStatsResponse,
|
||||||
|
UpdateTaskRequest,
|
||||||
},
|
},
|
||||||
routes::{
|
routes::{
|
||||||
__path_create_task, __path_delete_task, __path_disable_task, __path_enable_task,
|
__path_create_task, __path_delete_task, __path_disable_task, __path_enable_task,
|
||||||
@@ -162,6 +163,47 @@ pub async fn list_directories(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/api/v1/stats",
|
||||||
|
tag = "tasks",
|
||||||
|
responses((status = 200, description = "Dashboard counters", body = TaskStatsResponse))
|
||||||
|
)]
|
||||||
|
pub async fn task_stats(State(state): State<AppState>) -> ApiResult<Json<TaskStatsResponse>> {
|
||||||
|
let counts = crate::tasks::repository::counts(&state.pool).await?;
|
||||||
|
Ok(Json(TaskStatsResponse {
|
||||||
|
total: counts.total,
|
||||||
|
enabled: counts.enabled,
|
||||||
|
running: u64::try_from(state.executor.running_count()).unwrap_or_default(),
|
||||||
|
script: counts.script,
|
||||||
|
http: counts.http,
|
||||||
|
process: counts.process,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/api/v1/schedule",
|
||||||
|
tag = "tasks",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Next trigger time per enabled task", body = Vec<ScheduledTaskResponse>)
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn list_schedule(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
) -> ApiResult<Json<Vec<ScheduledTaskResponse>>> {
|
||||||
|
let tasks = crate::tasks::repository::list_enabled_tasks(&state.pool).await?;
|
||||||
|
Ok(Json(
|
||||||
|
tasks
|
||||||
|
.into_iter()
|
||||||
|
.map(|task| ScheduledTaskResponse {
|
||||||
|
next_run_at: state.scheduler.next_run_at(&task.id),
|
||||||
|
task_id: task.id,
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(OpenApi)]
|
#[derive(OpenApi)]
|
||||||
#[openapi(
|
#[openapi(
|
||||||
info(
|
info(
|
||||||
@@ -175,6 +217,8 @@ pub async fn list_directories(
|
|||||||
preview_cron,
|
preview_cron,
|
||||||
probe_python_venv,
|
probe_python_venv,
|
||||||
list_directories,
|
list_directories,
|
||||||
|
task_stats,
|
||||||
|
list_schedule,
|
||||||
list_tasks,
|
list_tasks,
|
||||||
create_task,
|
create_task,
|
||||||
get_task,
|
get_task,
|
||||||
@@ -202,6 +246,8 @@ pub async fn list_directories(
|
|||||||
InterpreterKind,
|
InterpreterKind,
|
||||||
DirectoryEntry,
|
DirectoryEntry,
|
||||||
DirectoryListingResponse,
|
DirectoryListingResponse,
|
||||||
|
TaskStatsResponse,
|
||||||
|
ScheduledTaskResponse,
|
||||||
ErrorResponse,
|
ErrorResponse,
|
||||||
TaskActionType,
|
TaskActionType,
|
||||||
HttpMethod,
|
HttpMethod,
|
||||||
@@ -252,10 +298,14 @@ pub fn router(state: AppState) -> Router {
|
|||||||
axum::routing::post(probe_python_venv),
|
axum::routing::post(probe_python_venv),
|
||||||
)
|
)
|
||||||
.route("/api/v1/system/directories", get(list_directories))
|
.route("/api/v1/system/directories", get(list_directories))
|
||||||
|
.route("/api/v1/stats", get(task_stats))
|
||||||
|
.route("/api/v1/schedule", get(list_schedule))
|
||||||
.route("/api/v1/openapi.json", get(openapi_json))
|
.route("/api/v1/openapi.json", get(openapi_json))
|
||||||
.merge(crate::tasks::routes::router())
|
.merge(crate::tasks::routes::router())
|
||||||
.merge(crate::runs::routes::router())
|
.merge(crate::runs::routes::router())
|
||||||
.route("/api/{*path}", any(api_not_found))
|
.route("/api/{*path}", any(api_not_found))
|
||||||
|
// 本地工具也不需要接受超大请求体;任务载荷与日志上传都用不到 2 MiB 以上。
|
||||||
|
.layer(DefaultBodyLimit::max(2 * 1024 * 1024))
|
||||||
.layer(cors)
|
.layer(cors)
|
||||||
.route_layer(middleware::from_fn_with_state(
|
.route_layer(middleware::from_fn_with_state(
|
||||||
state.clone(),
|
state.clone(),
|
||||||
|
|||||||
+16
-9
@@ -1,20 +1,27 @@
|
|||||||
use std::{error::Error, str::FromStr};
|
use std::{error::Error, str::FromStr, time::Duration};
|
||||||
|
|
||||||
use sqlx::{
|
use sqlx::{
|
||||||
SqlitePool,
|
SqlitePool,
|
||||||
migrate::MigrateError,
|
migrate::MigrateError,
|
||||||
sqlite::{SqliteConnectOptions, SqlitePoolOptions},
|
sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub async fn connect(database_url: &str) -> Result<SqlitePool, Box<dyn Error>> {
|
pub async fn connect(database_url: &str) -> Result<SqlitePool, Box<dyn Error>> {
|
||||||
let options = SqliteConnectOptions::from_str(database_url)?
|
let in_memory = database_url.contains(":memory:");
|
||||||
|
let mut options = SqliteConnectOptions::from_str(database_url)?
|
||||||
.create_if_missing(true)
|
.create_if_missing(true)
|
||||||
.foreign_keys(true);
|
.foreign_keys(true)
|
||||||
let max_connections = if database_url == "sqlite::memory:" {
|
.busy_timeout(Duration::from_secs(5));
|
||||||
1
|
|
||||||
} else {
|
// 运行日志和状态写入与前端查询会并发访问同一个库;WAL 允许读写并行,
|
||||||
5
|
// busy_timeout 则避免瞬时冲突直接返回 "database is locked"。
|
||||||
};
|
if !in_memory {
|
||||||
|
options = options
|
||||||
|
.journal_mode(SqliteJournalMode::Wal)
|
||||||
|
.synchronous(SqliteSynchronous::Normal);
|
||||||
|
}
|
||||||
|
|
||||||
|
let max_connections = if in_memory { 1 } else { 5 };
|
||||||
let pool = SqlitePoolOptions::new()
|
let pool = SqlitePoolOptions::new()
|
||||||
.max_connections(max_connections)
|
.max_connections(max_connections)
|
||||||
.connect_with(options)
|
.connect_with(options)
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ pub fn prepare_script_environment(
|
|||||||
.to_string_lossy()
|
.to_string_lossy()
|
||||||
.into_owned(),
|
.into_owned(),
|
||||||
);
|
);
|
||||||
environment.insert("PYTHONUNBUFFERED".to_owned(), "1".to_owned());
|
apply_stdio_defaults(&mut environment);
|
||||||
return Ok(PreparedPython {
|
return Ok(PreparedPython {
|
||||||
interpreter: python,
|
interpreter: python,
|
||||||
virtual_env: Some(venv.clone()),
|
virtual_env: Some(venv.clone()),
|
||||||
@@ -93,7 +93,7 @@ pub fn prepare_script_environment(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
environment.insert("PYTHONUNBUFFERED".to_owned(), "1".to_owned());
|
apply_stdio_defaults(&mut environment);
|
||||||
Ok(PreparedPython {
|
Ok(PreparedPython {
|
||||||
interpreter: interpreter_path,
|
interpreter: interpreter_path,
|
||||||
virtual_env: None,
|
virtual_env: None,
|
||||||
@@ -102,6 +102,17 @@ pub fn prepare_script_environment(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Python 的标准流默认跟随系统代码页(简体中文为 GBK),一旦打印 emoji 或生僻字符
|
||||||
|
/// 就会 `UnicodeEncodeError` 直接失败。这里默认改用 UTF-8,日志侧本来就优先按 UTF-8 解码。
|
||||||
|
/// 用户在任务里显式设置的 `PYTHONIOENCODING` 仍然优先。
|
||||||
|
fn apply_stdio_defaults(environment: &mut BTreeMap<String, String>) {
|
||||||
|
// 实时日志依赖标准流不缓冲,这一项始终强制。
|
||||||
|
environment.insert("PYTHONUNBUFFERED".to_owned(), "1".to_owned());
|
||||||
|
environment
|
||||||
|
.entry("PYTHONIOENCODING".to_owned())
|
||||||
|
.or_insert_with(|| "utf-8".to_owned());
|
||||||
|
}
|
||||||
|
|
||||||
/// 虚拟环境的来源,用于在界面上说明"为什么"关联到了这个环境。
|
/// 虚拟环境的来源,用于在界面上说明"为什么"关联到了这个环境。
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -630,6 +641,48 @@ mod tests {
|
|||||||
assert!(prepared.remove_environment.contains(&"VIRTUAL_ENV_PROMPT"));
|
assert!(prepared.remove_environment.contains(&"VIRTUAL_ENV_PROMPT"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn injects_utf8_stdio_for_python_tasks() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
|
||||||
|
let prepared = prepare_script_environment(
|
||||||
|
temp.path().to_str().unwrap(),
|
||||||
|
"python.exe",
|
||||||
|
None,
|
||||||
|
&BTreeMap::new(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
prepared.environment.get("PYTHONIOENCODING"),
|
||||||
|
Some(&"utf-8".to_owned())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
prepared.environment.get("PYTHONUNBUFFERED"),
|
||||||
|
Some(&"1".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn keeps_user_pythonioencoding_override() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let mut user_env = BTreeMap::new();
|
||||||
|
user_env.insert("PYTHONIOENCODING".to_owned(), "gbk".to_owned());
|
||||||
|
|
||||||
|
let prepared = prepare_script_environment(
|
||||||
|
temp.path().to_str().unwrap(),
|
||||||
|
"python.exe",
|
||||||
|
None,
|
||||||
|
&user_env,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
prepared.environment.get("PYTHONIOENCODING"),
|
||||||
|
Some(&"gbk".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn probe(working_dir: &str, interpreter: &str, venv: Option<&str>) -> VenvProbeResponse {
|
fn probe(working_dir: &str, interpreter: &str, venv: Option<&str>) -> VenvProbeResponse {
|
||||||
probe_venv(&VenvProbeRequest {
|
probe_venv(&VenvProbeRequest {
|
||||||
working_dir: working_dir.to_owned(),
|
working_dir: working_dir.to_owned(),
|
||||||
|
|||||||
+176
-15
@@ -16,7 +16,7 @@ use reqwest::Method;
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use sqlx::SqlitePool;
|
use sqlx::SqlitePool;
|
||||||
use tokio::{
|
use tokio::{
|
||||||
io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader},
|
io::{AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader},
|
||||||
process::{Child, Command},
|
process::{Child, Command},
|
||||||
sync::{Mutex as AsyncMutex, broadcast},
|
sync::{Mutex as AsyncMutex, broadcast},
|
||||||
time::{Duration, sleep, timeout},
|
time::{Duration, sleep, timeout},
|
||||||
@@ -35,6 +35,7 @@ use crate::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
const MAX_LOG_BYTES: u64 = 5 * 1024 * 1024;
|
const MAX_LOG_BYTES: u64 = 5 * 1024 * 1024;
|
||||||
|
const MAX_LOG_LINE_BYTES: usize = 256 * 1024;
|
||||||
const MAX_HTTP_LOG_BODY_BYTES: usize = 64 * 1024;
|
const MAX_HTTP_LOG_BODY_BYTES: usize = 64 * 1024;
|
||||||
const TERMINATION_WAIT_SECS: u64 = 5;
|
const TERMINATION_WAIT_SECS: u64 = 5;
|
||||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||||
@@ -255,7 +256,7 @@ impl TaskExecutor {
|
|||||||
let run = match repository::create_queued(&self.inner.pool, &task.id, trigger).await {
|
let run = match repository::create_queued(&self.inner.pool, &task.id, trigger).await {
|
||||||
Ok(run) => run,
|
Ok(run) => run,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
lock_running(&self.inner.running).remove(&task.id);
|
remove_running_if(&self.inner.running, &task.id, None);
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -301,6 +302,11 @@ impl TaskExecutor {
|
|||||||
self.inner.events.subscribe()
|
self.inner.events.subscribe()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 当前正在执行的任务数量(用于统计接口)。
|
||||||
|
pub fn running_count(&self) -> usize {
|
||||||
|
lock_running(&self.inner.running).len()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn log_path(&self, task_id: &str, run_id: i64) -> PathBuf {
|
pub fn log_path(&self, task_id: &str, run_id: i64) -> PathBuf {
|
||||||
self.inner
|
self.inner
|
||||||
.logs_dir
|
.logs_dir
|
||||||
@@ -320,26 +326,52 @@ impl TaskExecutor {
|
|||||||
let _guard = self.inner.cleanup_lock.lock().await;
|
let _guard = self.inner.cleanup_lock.lock().await;
|
||||||
let logs_dir = self.inner.logs_dir.clone();
|
let logs_dir = self.inner.logs_dir.clone();
|
||||||
let config = self.inner.log_config.clone();
|
let config = self.inner.log_config.clone();
|
||||||
tokio::task::spawn_blocking(move || cleanup_logs_sync(&logs_dir, &config))
|
let active = self.active_log_paths();
|
||||||
|
tokio::task::spawn_blocking(move || cleanup_logs_sync(&logs_dir, &config, &active))
|
||||||
.await
|
.await
|
||||||
.map_err(std::io::Error::other)?
|
.map_err(std::io::Error::other)?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn active_log_paths(&self) -> std::collections::HashSet<PathBuf> {
|
||||||
|
lock_running(&self.inner.running)
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(task_id, handle)| {
|
||||||
|
handle.run_id.map(|run_id| {
|
||||||
|
self.inner
|
||||||
|
.logs_dir
|
||||||
|
.join(task_id)
|
||||||
|
.join(format!("{run_id}.log"))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_run(&self, task: TaskResponse, run_id: i64, cancellation: CancellationToken) {
|
async fn execute_run(&self, task: TaskResponse, run_id: i64, cancellation: CancellationToken) {
|
||||||
let started_at = Utc::now();
|
let started_at = Utc::now();
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
if let Err(error) = repository::mark_running(&self.inner.pool, run_id, started_at).await {
|
match repository::mark_running(&self.inner.pool, run_id, started_at).await {
|
||||||
tracing::error!(run_id, error = %error, "failed to mark task run as running");
|
Ok(true) => {}
|
||||||
lock_running(&self.inner.running).remove(&task.id);
|
Ok(false) => {
|
||||||
|
tracing::warn!(
|
||||||
|
run_id,
|
||||||
|
"run row disappeared before it could be marked running"
|
||||||
|
);
|
||||||
|
remove_running_if(&self.inner.running, &task.id, Some(run_id));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
Err(error) => {
|
||||||
|
tracing::error!(run_id, error = %error, "failed to mark task run as running");
|
||||||
|
remove_running_if(&self.inner.running, &task.id, Some(run_id));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
self.emit_run_by_id(run_id).await;
|
self.emit_run_by_id(run_id).await;
|
||||||
|
|
||||||
let outcome = self.run_action(&task, run_id, &cancellation).await;
|
let outcome = self.run_action(&task, run_id, &cancellation).await;
|
||||||
let finished_at = Utc::now();
|
let finished_at = Utc::now();
|
||||||
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||||
|
|
||||||
if let Err(error) = repository::complete(
|
match repository::complete(
|
||||||
&self.inner.pool,
|
&self.inner.pool,
|
||||||
run_id,
|
run_id,
|
||||||
outcome.status,
|
outcome.status,
|
||||||
@@ -350,11 +382,19 @@ impl TaskExecutor {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::error!(run_id, error = %error, "failed to complete task run");
|
Ok(true) => {}
|
||||||
|
Ok(false) => {
|
||||||
|
tracing::warn!(
|
||||||
|
run_id,
|
||||||
|
"run row disappeared before completion could be recorded"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(error) => tracing::error!(run_id, error = %error, "failed to complete task run"),
|
||||||
}
|
}
|
||||||
|
|
||||||
self.emit_run_by_id(run_id).await;
|
self.emit_run_by_id(run_id).await;
|
||||||
lock_running(&self.inner.running).remove(&task.id);
|
// 只清理属于自己的句柄:取消后立刻重跑时,旧 run 收尾不能删掉新 run 的闸门。
|
||||||
|
remove_running_if(&self.inner.running, &task.id, Some(run_id));
|
||||||
if let Err(error) = self.cleanup_logs().await {
|
if let Err(error) = self.cleanup_logs().await {
|
||||||
tracing::warn!(%error, "failed to clean task logs");
|
tracing::warn!(%error, "failed to clean task logs");
|
||||||
}
|
}
|
||||||
@@ -852,6 +892,21 @@ fn lock_running(
|
|||||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 只在句柄仍属于指定 run 时移除;`None` 表示"刚插入、还没拿到 run id"的占位句柄。
|
||||||
|
fn remove_running_if(
|
||||||
|
running: &Mutex<HashMap<String, RunningHandle>>,
|
||||||
|
task_id: &str,
|
||||||
|
run_id: Option<i64>,
|
||||||
|
) {
|
||||||
|
let mut running = lock_running(running);
|
||||||
|
if running
|
||||||
|
.get(task_id)
|
||||||
|
.is_some_and(|handle| handle.run_id == run_id)
|
||||||
|
{
|
||||||
|
running.remove(task_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn spawn_log_reader<R>(
|
fn spawn_log_reader<R>(
|
||||||
reader: R,
|
reader: R,
|
||||||
stream: &'static str,
|
stream: &'static str,
|
||||||
@@ -864,12 +919,17 @@ where
|
|||||||
let mut reader = BufReader::new(reader);
|
let mut reader = BufReader::new(reader);
|
||||||
let mut line = Vec::new();
|
let mut line = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
line.clear();
|
match read_line_capped(&mut reader, &mut line, MAX_LOG_LINE_BYTES).await {
|
||||||
match reader.read_until(b'\n', &mut line).await {
|
Ok((0, _)) => break,
|
||||||
Ok(0) => break,
|
Ok((_, truncated)) => {
|
||||||
Ok(_) => {
|
|
||||||
let text = sink.decode_output(&line);
|
let text = sink.decode_output(&line);
|
||||||
sink.append(&format!("[{stream}] {text}")).await;
|
let mut payload = format!("[{stream}] {text}");
|
||||||
|
if truncated {
|
||||||
|
payload.push_str(&format!(
|
||||||
|
"\n[system] 单行输出超过 {MAX_LOG_LINE_BYTES} 字节,已截断\n"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
sink.append(&payload).await;
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
tracing::warn!(stream, %error, "failed to read task output");
|
tracing::warn!(stream, %error, "failed to read task output");
|
||||||
@@ -880,6 +940,38 @@ where
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 读取一行,最多保留 `max` 字节。超出上限时丢弃剩余部分但继续读到换行,
|
||||||
|
/// 既保住内存上限,也不会让后续输出错位。
|
||||||
|
async fn read_line_capped<R>(
|
||||||
|
reader: &mut R,
|
||||||
|
buffer: &mut Vec<u8>,
|
||||||
|
max: usize,
|
||||||
|
) -> std::io::Result<(usize, bool)>
|
||||||
|
where
|
||||||
|
R: AsyncBufRead + Unpin,
|
||||||
|
{
|
||||||
|
buffer.clear();
|
||||||
|
let mut truncated = false;
|
||||||
|
loop {
|
||||||
|
let chunk = reader.fill_buf().await?;
|
||||||
|
if chunk.is_empty() {
|
||||||
|
return Ok((buffer.len(), truncated));
|
||||||
|
}
|
||||||
|
let newline = chunk.iter().position(|byte| *byte == b'\n');
|
||||||
|
let take = newline.map_or(chunk.len(), |index| index + 1);
|
||||||
|
let remaining = max.saturating_sub(buffer.len());
|
||||||
|
let copied = take.min(remaining);
|
||||||
|
buffer.extend_from_slice(&chunk[..copied]);
|
||||||
|
if copied < take {
|
||||||
|
truncated = true;
|
||||||
|
}
|
||||||
|
reader.consume(take);
|
||||||
|
if newline.is_some() {
|
||||||
|
return Ok((buffer.len(), truncated));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_after_termination(child: &mut Child) {
|
async fn wait_after_termination(child: &mut Child) {
|
||||||
if timeout(Duration::from_secs(TERMINATION_WAIT_SECS), child.wait())
|
if timeout(Duration::from_secs(TERMINATION_WAIT_SECS), child.wait())
|
||||||
.await
|
.await
|
||||||
@@ -935,13 +1027,19 @@ fn failed_outcome(message: impl Into<String>) -> ScriptOutcome {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cleanup_logs_sync(root: &Path, config: &LogRetentionConfig) -> std::io::Result<()> {
|
fn cleanup_logs_sync(
|
||||||
|
root: &Path,
|
||||||
|
config: &LogRetentionConfig,
|
||||||
|
active: &std::collections::HashSet<PathBuf>,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
if !root.exists() {
|
if !root.exists() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut files = Vec::new();
|
let mut files = Vec::new();
|
||||||
collect_log_files(root, &mut files)?;
|
collect_log_files(root, &mut files)?;
|
||||||
|
// 正在写入的日志不参与保留期/容量清理。
|
||||||
|
files.retain(|(path, _)| !active.contains(path));
|
||||||
let retention = Duration::from_secs(config.retention_days.saturating_mul(86_400));
|
let retention = Duration::from_secs(config.retention_days.saturating_mul(86_400));
|
||||||
let cutoff = SystemTime::now().checked_sub(retention);
|
let cutoff = SystemTime::now().checked_sub(retention);
|
||||||
|
|
||||||
@@ -1104,6 +1202,7 @@ mod tests {
|
|||||||
retention_days: 30,
|
retention_days: 30,
|
||||||
max_total_bytes: 10,
|
max_total_bytes: 10,
|
||||||
},
|
},
|
||||||
|
&std::collections::HashSet::new(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let total = std::fs::read_dir(temp.path())
|
let total = std::fs::read_dir(temp.path())
|
||||||
@@ -1112,4 +1211,66 @@ mod tests {
|
|||||||
.sum::<u64>();
|
.sum::<u64>();
|
||||||
assert!(total <= 10);
|
assert!(total <= 10);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_skips_logs_of_running_tasks() {
|
||||||
|
let temp = tempfile::TempDir::new().unwrap();
|
||||||
|
let active = temp.path().join("active.log");
|
||||||
|
std::fs::write(&active, vec![b'x'; 32]).unwrap();
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||||
|
let stale = temp.path().join("stale.log");
|
||||||
|
std::fs::write(&stale, vec![b'x'; 32]).unwrap();
|
||||||
|
|
||||||
|
let mut protected = std::collections::HashSet::new();
|
||||||
|
protected.insert(active.clone());
|
||||||
|
cleanup_logs_sync(
|
||||||
|
temp.path(),
|
||||||
|
&LogRetentionConfig {
|
||||||
|
retention_days: 0,
|
||||||
|
max_total_bytes: 1,
|
||||||
|
},
|
||||||
|
&protected,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(active.exists(), "active log must survive cleanup");
|
||||||
|
assert!(!stale.exists(), "stale log should be removed");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn capped_line_reader_truncates_but_keeps_reading() {
|
||||||
|
let payload = [b'a'; 64];
|
||||||
|
let mut reader = BufReader::new(&payload[..]);
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
|
||||||
|
let (bytes, truncated) = read_line_capped(&mut reader, &mut buffer, 16)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(bytes, 16);
|
||||||
|
assert!(truncated);
|
||||||
|
|
||||||
|
let (next, next_truncated) = read_line_capped(&mut reader, &mut buffer, 16)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(next, 0);
|
||||||
|
assert!(!next_truncated);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn removes_only_the_matching_run_handle() {
|
||||||
|
let running = Mutex::new(HashMap::from([(
|
||||||
|
"task".to_owned(),
|
||||||
|
RunningHandle {
|
||||||
|
run_id: Some(2),
|
||||||
|
cancellation: CancellationToken::new(),
|
||||||
|
},
|
||||||
|
)]));
|
||||||
|
|
||||||
|
remove_running_if(&running, "task", Some(1));
|
||||||
|
assert!(lock_running(&running).contains_key("task"));
|
||||||
|
|
||||||
|
remove_running_if(&running, "task", Some(2));
|
||||||
|
assert!(!lock_running(&running).contains_key("task"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,20 +119,22 @@ pub async fn create_skipped(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 返回是否真的更新到了运行记录;记录被删除(例如任务被删)时返回 false。
|
||||||
pub async fn mark_running(
|
pub async fn mark_running(
|
||||||
pool: &SqlitePool,
|
pool: &SqlitePool,
|
||||||
run_id: i64,
|
run_id: i64,
|
||||||
started_at: DateTime<Utc>,
|
started_at: DateTime<Utc>,
|
||||||
) -> ApiResult<()> {
|
) -> ApiResult<bool> {
|
||||||
sqlx::query("UPDATE task_runs SET status = ?, started_at = ? WHERE id = ?")
|
let result = sqlx::query("UPDATE task_runs SET status = ?, started_at = ? WHERE id = ?")
|
||||||
.bind(RunStatus::Running.as_str())
|
.bind(RunStatus::Running.as_str())
|
||||||
.bind(started_at)
|
.bind(started_at)
|
||||||
.bind(run_id)
|
.bind(run_id)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(result.rows_affected() > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 返回是否真的更新到了运行记录。
|
||||||
pub async fn complete(
|
pub async fn complete(
|
||||||
pool: &SqlitePool,
|
pool: &SqlitePool,
|
||||||
run_id: i64,
|
run_id: i64,
|
||||||
@@ -141,8 +143,8 @@ pub async fn complete(
|
|||||||
duration_ms: u64,
|
duration_ms: u64,
|
||||||
exit_code: Option<i32>,
|
exit_code: Option<i32>,
|
||||||
error_message: Option<&str>,
|
error_message: Option<&str>,
|
||||||
) -> ApiResult<()> {
|
) -> ApiResult<bool> {
|
||||||
sqlx::query(
|
let result = sqlx::query(
|
||||||
"UPDATE task_runs SET
|
"UPDATE task_runs SET
|
||||||
status = ?, finished_at = ?, duration_ms = ?,
|
status = ?, finished_at = ?, duration_ms = ?,
|
||||||
exit_code = ?, error_message = ?
|
exit_code = ?, error_message = ?
|
||||||
@@ -156,7 +158,22 @@ pub async fn complete(
|
|||||||
.bind(run_id)
|
.bind(run_id)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(result.rows_affected() > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 启动时收敛上一次进程遗留下来的非终态运行,避免界面永远显示"运行中"。
|
||||||
|
pub async fn fail_interrupted_runs(pool: &SqlitePool) -> ApiResult<u64> {
|
||||||
|
let result = sqlx::query(
|
||||||
|
"UPDATE task_runs SET status = ?, finished_at = ?, error_message = ?
|
||||||
|
WHERE status IN ('queued', 'running')",
|
||||||
|
)
|
||||||
|
.bind(RunStatus::Cancelled.as_str())
|
||||||
|
.bind(Utc::now())
|
||||||
|
.bind("TaskPilot 进程重启,运行被中断")
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(result.rows_affected())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get(pool: &SqlitePool, run_id: i64) -> ApiResult<Option<TaskRunResponse>> {
|
pub async fn get(pool: &SqlitePool, run_id: i64) -> ApiResult<Option<TaskRunResponse>> {
|
||||||
|
|||||||
+50
-12
@@ -14,18 +14,23 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use crate::{
|
use crate::{
|
||||||
error::ApiResult,
|
error::ApiResult,
|
||||||
runs::{executor::TaskExecutor, model::RunTrigger},
|
runs::{executor::TaskExecutor, model::RunTrigger},
|
||||||
tasks::{model::TaskAction, model::TaskResponse, repository},
|
tasks::{model::TaskResponse, repository},
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ScheduledTask {
|
struct ScheduledTask {
|
||||||
cancellation: CancellationToken,
|
cancellation: CancellationToken,
|
||||||
handle: JoinHandle<()>,
|
/// 只用来持有任务;取消时通过令牌让循环自行退出,不直接 abort。
|
||||||
|
/// 字段以下划线开头是因为它只承担"保持任务存活"的作用。
|
||||||
|
_handle: JoinHandle<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type NextRuns = Arc<Mutex<HashMap<String, DateTime<Utc>>>>;
|
||||||
|
|
||||||
struct SchedulerInner {
|
struct SchedulerInner {
|
||||||
executor: TaskExecutor,
|
executor: TaskExecutor,
|
||||||
shutdown: CancellationToken,
|
shutdown: CancellationToken,
|
||||||
jobs: Mutex<HashMap<String, ScheduledTask>>,
|
jobs: Mutex<HashMap<String, ScheduledTask>>,
|
||||||
|
next_runs: NextRuns,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -40,19 +45,22 @@ impl Scheduler {
|
|||||||
executor,
|
executor,
|
||||||
shutdown: CancellationToken::new(),
|
shutdown: CancellationToken::new(),
|
||||||
jobs: Mutex::new(HashMap::new()),
|
jobs: Mutex::new(HashMap::new()),
|
||||||
|
next_runs: Arc::new(Mutex::new(HashMap::new())),
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
for task in repository::list_enabled_script_tasks(&pool).await? {
|
for task in repository::list_enabled_tasks(&pool).await? {
|
||||||
scheduler.refresh(task);
|
scheduler.refresh(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(scheduler)
|
Ok(scheduler)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 重新注册任务:停用会取消调度,启用则按 cron 新建循环。
|
||||||
|
/// 三类 Action 都参与调度,执行器本身负责区分脚本、HTTP 与进程。
|
||||||
pub fn refresh(&self, task: TaskResponse) {
|
pub fn refresh(&self, task: TaskResponse) {
|
||||||
self.cancel(&task.id);
|
self.cancel(&task.id);
|
||||||
if !task.enabled || !matches!(task.action, TaskAction::Script { .. }) {
|
if !task.enabled {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,38 +69,52 @@ impl Scheduler {
|
|||||||
let executor = self.inner.executor.clone();
|
let executor = self.inner.executor.clone();
|
||||||
let shutdown = self.inner.shutdown.clone();
|
let shutdown = self.inner.shutdown.clone();
|
||||||
let cancellation_for_loop = cancellation.clone();
|
let cancellation_for_loop = cancellation.clone();
|
||||||
|
let next_runs = Arc::clone(&self.inner.next_runs);
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
schedule_loop(task_for_loop, executor, cancellation_for_loop, shutdown).await;
|
schedule_loop(
|
||||||
|
task_for_loop,
|
||||||
|
executor,
|
||||||
|
cancellation_for_loop,
|
||||||
|
shutdown,
|
||||||
|
next_runs,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
});
|
});
|
||||||
|
|
||||||
lock_jobs(&self.inner.jobs).insert(
|
lock_jobs(&self.inner.jobs).insert(
|
||||||
task.id,
|
task.id,
|
||||||
ScheduledTask {
|
ScheduledTask {
|
||||||
cancellation,
|
cancellation,
|
||||||
handle,
|
_handle: handle,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 取消注册。循环会在下一个 select 点退出,不会留下半途的 running 记录。
|
||||||
pub fn cancel(&self, task_id: &str) {
|
pub fn cancel(&self, task_id: &str) {
|
||||||
if let Some(job) = lock_jobs(&self.inner.jobs).remove(task_id) {
|
if let Some(job) = lock_jobs(&self.inner.jobs).remove(task_id) {
|
||||||
job.cancellation.cancel();
|
job.cancellation.cancel();
|
||||||
job.handle.abort();
|
|
||||||
}
|
}
|
||||||
|
lock_next_runs(&self.inner.next_runs).remove(task_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_scheduled(&self, task_id: &str) -> bool {
|
pub fn is_scheduled(&self, task_id: &str) -> bool {
|
||||||
lock_jobs(&self.inner.jobs).contains_key(task_id)
|
lock_jobs(&self.inner.jobs).contains_key(task_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 下次触发时间;未注册或刚取消时返回 None。
|
||||||
|
pub fn next_run_at(&self, task_id: &str) -> Option<DateTime<Utc>> {
|
||||||
|
lock_next_runs(&self.inner.next_runs).get(task_id).copied()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn shutdown(&self) {
|
pub fn shutdown(&self) {
|
||||||
self.inner.shutdown.cancel();
|
self.inner.shutdown.cancel();
|
||||||
let mut jobs = lock_jobs(&self.inner.jobs);
|
let mut jobs = lock_jobs(&self.inner.jobs);
|
||||||
for job in jobs.values() {
|
for job in jobs.values() {
|
||||||
job.cancellation.cancel();
|
job.cancellation.cancel();
|
||||||
job.handle.abort();
|
|
||||||
}
|
}
|
||||||
jobs.clear();
|
jobs.clear();
|
||||||
|
lock_next_runs(&self.inner.next_runs).clear();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,7 +127,6 @@ impl Drop for SchedulerInner {
|
|||||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
for job in jobs.values() {
|
for job in jobs.values() {
|
||||||
job.cancellation.cancel();
|
job.cancellation.cancel();
|
||||||
job.handle.abort();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -115,6 +136,7 @@ async fn schedule_loop(
|
|||||||
executor: TaskExecutor,
|
executor: TaskExecutor,
|
||||||
cancellation: CancellationToken,
|
cancellation: CancellationToken,
|
||||||
shutdown: CancellationToken,
|
shutdown: CancellationToken,
|
||||||
|
next_runs: NextRuns,
|
||||||
) {
|
) {
|
||||||
loop {
|
loop {
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
@@ -122,14 +144,22 @@ async fn schedule_loop(
|
|||||||
Ok(next) => next,
|
Ok(next) => next,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
tracing::error!(task_id = %task.id, %error, "cron schedule calculation failed");
|
tracing::error!(task_id = %task.id, %error, "cron schedule calculation failed");
|
||||||
|
lock_next_runs(&next_runs).remove(&task.id);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
lock_next_runs(&next_runs).insert(task.id.clone(), next);
|
||||||
let delay = (next - now).to_std().unwrap_or_default();
|
let delay = (next - now).to_std().unwrap_or_default();
|
||||||
|
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = cancellation.cancelled() => return,
|
_ = cancellation.cancelled() => {
|
||||||
_ = shutdown.cancelled() => return,
|
lock_next_runs(&next_runs).remove(&task.id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
_ = shutdown.cancelled() => {
|
||||||
|
lock_next_runs(&next_runs).remove(&task.id);
|
||||||
|
return;
|
||||||
|
}
|
||||||
_ = tokio::time::sleep(delay) => {
|
_ = tokio::time::sleep(delay) => {
|
||||||
if let Err(error) = executor.trigger(&task, RunTrigger::Schedule).await {
|
if let Err(error) = executor.trigger(&task, RunTrigger::Schedule).await {
|
||||||
tracing::error!(task_id = %task.id, %error, "scheduled task trigger failed");
|
tracing::error!(task_id = %task.id, %error, "scheduled task trigger failed");
|
||||||
@@ -156,11 +186,19 @@ fn lock_jobs(
|
|||||||
jobs.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
jobs.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn lock_next_runs(
|
||||||
|
next_runs: &Mutex<HashMap<String, DateTime<Utc>>>,
|
||||||
|
) -> MutexGuard<'_, HashMap<String, DateTime<Utc>>> {
|
||||||
|
next_runs
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use chrono::{TimeZone, Timelike};
|
use chrono::{TimeZone, Timelike};
|
||||||
|
|
||||||
use crate::tasks::model::{ConcurrencyPolicy, RetryPolicy};
|
use crate::tasks::model::{ConcurrencyPolicy, RetryPolicy, TaskAction};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
|||||||
@@ -33,6 +33,13 @@ impl AppState {
|
|||||||
log_config.max_total_bytes = megabytes.saturating_mul(1024 * 1024);
|
log_config.max_total_bytes = megabytes.saturating_mul(1024 * 1024);
|
||||||
}
|
}
|
||||||
let executor = TaskExecutor::new(pool.clone(), logs_dir, log_config);
|
let executor = TaskExecutor::new(pool.clone(), logs_dir, log_config);
|
||||||
|
let reconciled = crate::runs::repository::fail_interrupted_runs(&pool).await?;
|
||||||
|
if reconciled > 0 {
|
||||||
|
tracing::warn!(
|
||||||
|
count = reconciled,
|
||||||
|
"marked runs interrupted by the previous shutdown as cancelled"
|
||||||
|
);
|
||||||
|
}
|
||||||
executor.cleanup_logs().await.map_err(ApiError::internal)?;
|
executor.cleanup_logs().await.map_err(ApiError::internal)?;
|
||||||
let scheduler = Scheduler::start(pool.clone(), executor.clone()).await?;
|
let scheduler = Scheduler::start(pool.clone(), executor.clone()).await?;
|
||||||
|
|
||||||
|
|||||||
@@ -176,3 +176,28 @@ pub struct ListTasksQuery {
|
|||||||
pub page: Option<u32>,
|
pub page: Option<u32>,
|
||||||
pub page_size: Option<u32>,
|
pub page_size: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 仪表盘指标:一次请求拿到全部计数,避免为每个指标各发一次列表查询。
|
||||||
|
#[derive(Debug, Serialize, ToSchema)]
|
||||||
|
pub struct TaskStatsResponse {
|
||||||
|
/// 任务总数
|
||||||
|
pub total: u64,
|
||||||
|
/// 已启用任务数
|
||||||
|
pub enabled: u64,
|
||||||
|
/// 当前正在执行的任务数
|
||||||
|
pub running: u64,
|
||||||
|
/// 脚本类任务数
|
||||||
|
pub script: u64,
|
||||||
|
/// HTTP 类任务数
|
||||||
|
pub http: u64,
|
||||||
|
/// 进程类任务数
|
||||||
|
pub process: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 已注册调度的任务及其下次触发时间。
|
||||||
|
#[derive(Debug, Serialize, ToSchema)]
|
||||||
|
pub struct ScheduledTaskResponse {
|
||||||
|
pub task_id: String,
|
||||||
|
/// 下次触发时间;调度正在重建时可能为空
|
||||||
|
pub next_run_at: Option<DateTime<Utc>>,
|
||||||
|
}
|
||||||
|
|||||||
@@ -15,6 +15,38 @@ const TASK_COLUMNS: &str = "
|
|||||||
created_at, updated_at
|
created_at, updated_at
|
||||||
";
|
";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
|
pub struct TaskCounts {
|
||||||
|
pub total: u64,
|
||||||
|
pub enabled: u64,
|
||||||
|
pub script: u64,
|
||||||
|
pub http: u64,
|
||||||
|
pub process: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 单条 SQL 统计全部指标,供仪表盘使用。
|
||||||
|
pub async fn counts(pool: &SqlitePool) -> ApiResult<TaskCounts> {
|
||||||
|
let (total, enabled, script, http, process): (i64, i64, i64, i64, i64) = sqlx::query_as(
|
||||||
|
"SELECT
|
||||||
|
COUNT(*),
|
||||||
|
COALESCE(SUM(enabled), 0),
|
||||||
|
COALESCE(SUM(action_type = 'script'), 0),
|
||||||
|
COALESCE(SUM(action_type = 'http'), 0),
|
||||||
|
COALESCE(SUM(action_type = 'process'), 0)
|
||||||
|
FROM tasks",
|
||||||
|
)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(TaskCounts {
|
||||||
|
total: u64::try_from(total).unwrap_or_default(),
|
||||||
|
enabled: u64::try_from(enabled).unwrap_or_default(),
|
||||||
|
script: u64::try_from(script).unwrap_or_default(),
|
||||||
|
http: u64::try_from(http).unwrap_or_default(),
|
||||||
|
process: u64::try_from(process).unwrap_or_default(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub struct ListFilters {
|
pub struct ListFilters {
|
||||||
pub q: Option<String>,
|
pub q: Option<String>,
|
||||||
pub enabled: Option<bool>,
|
pub enabled: Option<bool>,
|
||||||
@@ -233,14 +265,15 @@ pub async fn list(pool: &SqlitePool, filters: &ListFilters) -> ApiResult<TaskLis
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_enabled_script_tasks(pool: &SqlitePool) -> ApiResult<Vec<TaskResponse>> {
|
/// 启动时恢复调度用的列表:所有启用任务都参与调度,不再只限脚本。
|
||||||
|
pub async fn list_enabled_tasks(pool: &SqlitePool) -> ApiResult<Vec<TaskResponse>> {
|
||||||
let rows = sqlx::query_as::<_, TaskRow>(
|
let rows = sqlx::query_as::<_, TaskRow>(
|
||||||
"SELECT
|
"SELECT
|
||||||
id, name, cron, timezone, enabled, timeout_secs, concurrency_policy,
|
id, name, cron, timezone, enabled, timeout_secs, concurrency_policy,
|
||||||
retry_max_attempts, retry_delay_secs, action_type, action_json,
|
retry_max_attempts, retry_delay_secs, action_type, action_json,
|
||||||
created_at, updated_at
|
created_at, updated_at
|
||||||
FROM tasks
|
FROM tasks
|
||||||
WHERE enabled = 1 AND action_type = 'script'
|
WHERE enabled = 1
|
||||||
ORDER BY updated_at DESC, id ASC",
|
ORDER BY updated_at DESC, id ASC",
|
||||||
)
|
)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
|
|||||||
+148
-2
@@ -573,7 +573,12 @@ async fn overlapping_trigger_is_skipped_and_timeout_kills_process_tree() {
|
|||||||
&working_dir,
|
&working_dir,
|
||||||
"powershell.exe",
|
"powershell.exe",
|
||||||
"-NoProfile",
|
"-NoProfile",
|
||||||
vec!["-File", &script_path.to_string_lossy()],
|
vec![
|
||||||
|
"-ExecutionPolicy",
|
||||||
|
"Bypass",
|
||||||
|
"-File",
|
||||||
|
&script_path.to_string_lossy(),
|
||||||
|
],
|
||||||
"0 0 2 * * *",
|
"0 0 2 * * *",
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
@@ -603,7 +608,10 @@ async fn overlapping_trigger_is_skipped_and_timeout_kills_process_tree() {
|
|||||||
assert_eq!(second_run["status"], "skipped");
|
assert_eq!(second_run["status"], "skipped");
|
||||||
|
|
||||||
let completed = wait_for_run(&app.router, first_run_id).await;
|
let completed = wait_for_run(&app.router, first_run_id).await;
|
||||||
assert_eq!(completed["status"], "timed_out");
|
assert_eq!(
|
||||||
|
completed["status"], "timed_out",
|
||||||
|
"unexpected run: {completed}"
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
completed["error_message"]
|
completed["error_message"]
|
||||||
.as_str()
|
.as_str()
|
||||||
@@ -821,3 +829,141 @@ async fn long_utf8_log_is_reassembled_across_chunk_boundaries() {
|
|||||||
);
|
);
|
||||||
app.state.shutdown();
|
app.state.shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stats_endpoint_aggregates_task_counts() {
|
||||||
|
let app = test_app().await;
|
||||||
|
let working_dir = app._temp_dir.path().to_string_lossy().into_owned();
|
||||||
|
let script = script_task(
|
||||||
|
"stats_script",
|
||||||
|
&working_dir,
|
||||||
|
"cmd.exe",
|
||||||
|
"/C",
|
||||||
|
vec!["exit", "0"],
|
||||||
|
"0 0 2 * * *",
|
||||||
|
5,
|
||||||
|
);
|
||||||
|
let http = http_task("http://127.0.0.1:9/never");
|
||||||
|
for payload in [script.clone(), http.clone()] {
|
||||||
|
let response = send_json(&app.router, Method::POST, "/api/v1/tasks", Some(payload)).await;
|
||||||
|
assert_eq!(response.status(), StatusCode::CREATED);
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = send_json(&app.router, Method::GET, "/api/v1/stats", None).await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let stats = response_json(response).await;
|
||||||
|
|
||||||
|
assert_eq!(stats["total"], 2);
|
||||||
|
assert_eq!(stats["enabled"], 2);
|
||||||
|
assert_eq!(stats["script"], 1);
|
||||||
|
assert_eq!(stats["http"], 1);
|
||||||
|
assert_eq!(stats["process"], 0);
|
||||||
|
assert_eq!(stats["running"], 0);
|
||||||
|
app.state.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn schedule_endpoint_reports_next_run_for_enabled_tasks() {
|
||||||
|
let app = test_app().await;
|
||||||
|
let working_dir = app._temp_dir.path().to_string_lossy().into_owned();
|
||||||
|
let task = script_task(
|
||||||
|
"scheduled_stats",
|
||||||
|
&working_dir,
|
||||||
|
"cmd.exe",
|
||||||
|
"/C",
|
||||||
|
vec!["exit", "0"],
|
||||||
|
"*/30 * * * * *",
|
||||||
|
5,
|
||||||
|
);
|
||||||
|
let created = send_json(&app.router, Method::POST, "/api/v1/tasks", Some(task)).await;
|
||||||
|
assert_eq!(created.status(), StatusCode::CREATED);
|
||||||
|
|
||||||
|
let response = send_json(&app.router, Method::GET, "/api/v1/schedule", None).await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
let schedule = response_json(response).await;
|
||||||
|
let entries = schedule.as_array().expect("schedule should be an array");
|
||||||
|
let entry = entries
|
||||||
|
.iter()
|
||||||
|
.find(|item| item["task_id"] == "scheduled_stats")
|
||||||
|
.expect("enabled task should appear in the schedule");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
entry["next_run_at"].is_string(),
|
||||||
|
"next_run_at should be computed for a registered task"
|
||||||
|
);
|
||||||
|
app.state.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn http_tasks_are_registered_for_scheduling() {
|
||||||
|
let app = test_app().await;
|
||||||
|
let task = http_task("http://127.0.0.1:9/never");
|
||||||
|
let created = send_json(&app.router, Method::POST, "/api/v1/tasks", Some(task)).await;
|
||||||
|
assert_eq!(created.status(), StatusCode::CREATED);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
app.state.scheduler.is_scheduled("http_task"),
|
||||||
|
"HTTP tasks must take part in scheduled runs"
|
||||||
|
);
|
||||||
|
|
||||||
|
let disabled = send_json(
|
||||||
|
&app.router,
|
||||||
|
Method::POST,
|
||||||
|
"/api/v1/tasks/http_task/disable",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(disabled.status(), StatusCode::OK);
|
||||||
|
assert!(!app.state.scheduler.is_scheduled("http_task"));
|
||||||
|
app.state.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn startup_marks_runs_interrupted_by_restart_as_cancelled() {
|
||||||
|
let app = test_app().await;
|
||||||
|
let working_dir = app._temp_dir.path().to_string_lossy().into_owned();
|
||||||
|
let task = script_task(
|
||||||
|
"interrupted_task",
|
||||||
|
&working_dir,
|
||||||
|
"cmd.exe",
|
||||||
|
"/C",
|
||||||
|
vec!["exit", "0"],
|
||||||
|
"0 0 2 * * *",
|
||||||
|
5,
|
||||||
|
);
|
||||||
|
let created = send_json(&app.router, Method::POST, "/api/v1/tasks", Some(task)).await;
|
||||||
|
assert_eq!(created.status(), StatusCode::CREATED);
|
||||||
|
|
||||||
|
// 模拟上一次进程被强杀后遗留的运行记录。
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO task_runs (task_id, trigger_type, status, created_at)
|
||||||
|
VALUES ('interrupted_task', 'manual', 'running', ?)",
|
||||||
|
)
|
||||||
|
.bind(chrono::Utc::now())
|
||||||
|
.execute(&app.state.pool)
|
||||||
|
.await
|
||||||
|
.expect("stale run should be inserted");
|
||||||
|
|
||||||
|
let reconciled = task_pilot_backend::runs::repository::fail_interrupted_runs(&app.state.pool)
|
||||||
|
.await
|
||||||
|
.expect("reconciliation should run");
|
||||||
|
assert_eq!(reconciled, 1);
|
||||||
|
|
||||||
|
let runs = send_json(
|
||||||
|
&app.router,
|
||||||
|
Method::GET,
|
||||||
|
"/api/v1/tasks/interrupted_task/runs",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let body = response_json(runs).await;
|
||||||
|
let item = &body["items"][0];
|
||||||
|
assert_eq!(item["status"], "cancelled");
|
||||||
|
assert!(
|
||||||
|
item["error_message"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("重启")
|
||||||
|
);
|
||||||
|
app.state.shutdown();
|
||||||
|
}
|
||||||
|
|||||||
@@ -317,6 +317,29 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/v1/schedule": {
|
||||||
|
"get": {
|
||||||
|
"tags": [
|
||||||
|
"tasks"
|
||||||
|
],
|
||||||
|
"operationId": "list_schedule",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "Next trigger time per enabled task",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/components/schemas/ScheduledTaskResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/v1/session": {
|
"/api/v1/session": {
|
||||||
"get": {
|
"get": {
|
||||||
"tags": [
|
"tags": [
|
||||||
@@ -337,6 +360,26 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/v1/stats": {
|
||||||
|
"get": {
|
||||||
|
"tags": [
|
||||||
|
"tasks"
|
||||||
|
],
|
||||||
|
"operationId": "task_stats",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "Dashboard counters",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/TaskStatsResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/v1/system/directories": {
|
"/api/v1/system/directories": {
|
||||||
"get": {
|
"get": {
|
||||||
"tags": [
|
"tags": [
|
||||||
@@ -1088,6 +1131,26 @@
|
|||||||
"retry"
|
"retry"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"ScheduledTaskResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "已注册调度的任务及其下次触发时间。",
|
||||||
|
"required": [
|
||||||
|
"task_id"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"next_run_at": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "下次触发时间;调度正在重建时可能为空"
|
||||||
|
},
|
||||||
|
"task_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"SessionResponse": {
|
"SessionResponse": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
@@ -1426,6 +1489,56 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"TaskStatsResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "仪表盘指标:一次请求拿到全部计数,避免为每个指标各发一次列表查询。",
|
||||||
|
"required": [
|
||||||
|
"total",
|
||||||
|
"enabled",
|
||||||
|
"running",
|
||||||
|
"script",
|
||||||
|
"http",
|
||||||
|
"process"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"enabled": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "已启用任务数",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"http": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "HTTP 类任务数",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"process": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "进程类任务数",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"running": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "当前正在执行的任务数",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"script": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "脚本类任务数",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"total": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64",
|
||||||
|
"description": "任务总数",
|
||||||
|
"minimum": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"UpdateTaskRequest": {
|
"UpdateTaskRequest": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# ADR-0007:运行期可靠性优化
|
||||||
|
|
||||||
|
- 状态:已接受
|
||||||
|
- 日期:2026-10-25
|
||||||
|
|
||||||
|
## 背景
|
||||||
|
|
||||||
|
代码评审与端到端自检暴露出五类问题,都会让"看起来正常"的功能在边界条件下失真:
|
||||||
|
|
||||||
|
1. 取消任务时执行器直接删除 `running` 句柄,而旧运行的收尾逻辑又无条件删除同任务句柄。取消后立刻重跑时,旧运行会把新运行的闸门删掉,跳过策略失效,同一任务可能并发执行。
|
||||||
|
2. 调度器用 `JoinHandle::abort()` 取消循环。若恰好卡在 `executor.trigger()` 的数据库 await 中,`running` 表项和 `queued` 运行会永久残留,该任务此后永远被判定为"运行中"并跳过。
|
||||||
|
3. 进程被强杀后,数据库里的 `queued`/`running` 运行没有对账,前端会一直显示"运行中"并持续轮询。
|
||||||
|
4. `mark_running`/`complete` 不检查 `rows_affected`,运行记录被级联删除后收尾静默失效。
|
||||||
|
5. 调度器只注册脚本任务,HTTP 与进程任务"启用"后返回 200 却永远不会被定时触发。
|
||||||
|
|
||||||
|
## 决策
|
||||||
|
|
||||||
|
1. 句柄带运行 ID,收尾使用 `remove_running_if(task_id, run_id)`:只有仍属于本次运行的句柄才会被移除;取消时依旧立即释放闸门,允许马上重跑。
|
||||||
|
2. 调度循环只用 `CancellationToken` 退出,不再 `abort`;`ScheduledTask` 的 `JoinHandle` 仅用于保活。
|
||||||
|
3. 启动时执行 `fail_interrupted_runs`:把遗留的 `queued`/`running` 统一改为 `cancelled`,并写入"进程重启,运行被中断"。
|
||||||
|
4. `mark_running`/`complete` 返回是否更新成功,执行器对 0 行更新记警告并同步清理内存句柄。
|
||||||
|
5. 三类 Action 统一参与调度:`list_enabled_tasks` 不再按 `action_type` 过滤,`Scheduler::refresh` 只判断 `enabled`。
|
||||||
|
6. SQLite 启用 WAL + `busy_timeout(5s)` + `synchronous=NORMAL`,避免执行器写日志时前端查询撞上 `database is locked`。
|
||||||
|
7. 日志清理跳过正在写入的文件;单行输出上限 256 KiB,超出后截断并继续读行尾。
|
||||||
|
8. 请求体上限 2 MiB;新增 `/api/v1/stats`(一次查询拿到全部计数)与 `/api/v1/schedule`(下次触发时间)。
|
||||||
|
9. 前端缓存会话令牌,写请求收到 403 时丢弃缓存并重试一次(覆盖后端重启轮换令牌)。
|
||||||
|
10. Python 任务默认注入 `PYTHONIOENCODING=utf-8`(用户显式设置优先),`PYTHONUNBUFFERED=1` 保持强制。
|
||||||
|
|
||||||
|
## 原因
|
||||||
|
|
||||||
|
- 句柄竞态与 `abort` 泄漏都会造成"任务不再执行"且无任何报错,是排障成本最高的一类缺陷。
|
||||||
|
- 对账让崩溃恢复变成确定行为,前端轮询也能正常收敛。
|
||||||
|
- 统一调度消除了"启用但永不触发"的假能力。
|
||||||
|
- 统计与调度接口把仪表盘从 4 次请求降到 2 次,并让"下次执行时间"成为可见信息。
|
||||||
|
|
||||||
|
## 边界
|
||||||
|
|
||||||
|
- 取消后立即可重跑,意味着旧进程的终止与新进程的启动可能短暂重叠(取决于 Job Object 终止速度)。
|
||||||
|
- 没有全局并发上限:多个任务在同一时刻触发会各自启动一个进程。
|
||||||
|
- 对账只针对 `queued`/`running`,不会尝试恢复重试或补偿错过的调度。
|
||||||
|
- 单行截断后该行内容不完整,日志里会多一条 `[system]` 说明。
|
||||||
@@ -68,6 +68,21 @@ function createFetchMock(options: FetchMockOptions = {}) {
|
|||||||
if (url.endsWith('/api/v1/session')) {
|
if (url.endsWith('/api/v1/session')) {
|
||||||
return jsonResponse({ token: 'test-token' })
|
return jsonResponse({ token: 'test-token' })
|
||||||
}
|
}
|
||||||
|
if (url.endsWith('/api/v1/stats')) {
|
||||||
|
return jsonResponse({
|
||||||
|
total: 1,
|
||||||
|
enabled: 1,
|
||||||
|
running: 0,
|
||||||
|
script: 1,
|
||||||
|
http: 0,
|
||||||
|
process: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (url.endsWith('/api/v1/schedule')) {
|
||||||
|
return jsonResponse([
|
||||||
|
{ task_id: 'echo_task', next_run_at: '2026-10-19T02:00:00Z' },
|
||||||
|
])
|
||||||
|
}
|
||||||
if (url.endsWith('/api/v1/cron/preview')) {
|
if (url.endsWith('/api/v1/cron/preview')) {
|
||||||
return jsonResponse({
|
return jsonResponse({
|
||||||
timezone: 'Asia/Shanghai',
|
timezone: 'Asia/Shanghai',
|
||||||
@@ -152,6 +167,13 @@ describe('TaskDashboard', () => {
|
|||||||
expect(await screen.findByText('在线')).toBeInTheDocument()
|
expect(await screen.findByText('在线')).toBeInTheDocument()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('shows the next run time reported by the schedule endpoint', async () => {
|
||||||
|
vi.stubGlobal('fetch', createFetchMock())
|
||||||
|
renderApp()
|
||||||
|
|
||||||
|
expect(await screen.findByText(/下次/)).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
|
||||||
it('creates a script task with the visible form values', async () => {
|
it('creates a script task with the visible form values', async () => {
|
||||||
const fetchMock = createFetchMock()
|
const fetchMock = createFetchMock()
|
||||||
vi.stubGlobal('fetch', fetchMock)
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ export type InterpreterKind = components['schemas']['InterpreterKind']
|
|||||||
export type DirectoryEntry = components['schemas']['DirectoryEntry']
|
export type DirectoryEntry = components['schemas']['DirectoryEntry']
|
||||||
export type DirectoryListingResponse =
|
export type DirectoryListingResponse =
|
||||||
components['schemas']['DirectoryListingResponse']
|
components['schemas']['DirectoryListingResponse']
|
||||||
|
export type TaskStats = components['schemas']['TaskStatsResponse']
|
||||||
|
export type ScheduledTask = components['schemas']['ScheduledTaskResponse']
|
||||||
|
|
||||||
export interface TaskFilters {
|
export interface TaskFilters {
|
||||||
q?: string
|
q?: string
|
||||||
@@ -77,20 +79,27 @@ async function requestJson<T>(
|
|||||||
path: string,
|
path: string,
|
||||||
init: RequestInit = {},
|
init: RequestInit = {},
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
|
const method = (init.method ?? 'GET').toUpperCase()
|
||||||
|
const mutating = !['GET', 'HEAD', 'OPTIONS'].includes(method)
|
||||||
|
|
||||||
|
const send = async (): Promise<Response> => {
|
||||||
const headers = new Headers(init.headers)
|
const headers = new Headers(init.headers)
|
||||||
headers.set('Accept', 'application/json')
|
headers.set('Accept', 'application/json')
|
||||||
const method = (init.method ?? 'GET').toUpperCase()
|
|
||||||
if (init.body) {
|
if (init.body) {
|
||||||
headers.set('Content-Type', 'application/json')
|
headers.set('Content-Type', 'application/json')
|
||||||
}
|
}
|
||||||
if (!['GET', 'HEAD', 'OPTIONS'].includes(method)) {
|
if (mutating) {
|
||||||
headers.set('X-TaskPilot-Token', await fetchSessionToken())
|
headers.set('X-TaskPilot-Token', await fetchSessionToken())
|
||||||
}
|
}
|
||||||
|
return fetch(path, { ...init, headers })
|
||||||
|
}
|
||||||
|
|
||||||
const response = await fetch(path, {
|
let response = await send()
|
||||||
...init,
|
if (mutating && response.status === 403) {
|
||||||
headers,
|
// 后端重启会轮换管理令牌:丢弃缓存后重试一次。
|
||||||
})
|
invalidateSessionToken()
|
||||||
|
response = await send()
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
let body: ErrorResponse | undefined
|
let body: ErrorResponse | undefined
|
||||||
@@ -108,15 +117,34 @@ async function requestJson<T>(
|
|||||||
return (await response.json()) as T
|
return (await response.json()) as T
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let cachedSessionToken: string | null = null
|
||||||
|
let pendingSessionToken: Promise<string> | null = null
|
||||||
|
|
||||||
|
export function invalidateSessionToken(): void {
|
||||||
|
cachedSessionToken = null
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchSessionToken(): Promise<string> {
|
async function fetchSessionToken(): Promise<string> {
|
||||||
const response = await fetch('/api/v1/session', {
|
if (cachedSessionToken) {
|
||||||
|
return cachedSessionToken
|
||||||
|
}
|
||||||
|
if (!pendingSessionToken) {
|
||||||
|
pendingSessionToken = fetch('/api/v1/session', {
|
||||||
headers: { Accept: 'application/json' },
|
headers: { Accept: 'application/json' },
|
||||||
})
|
})
|
||||||
|
.then(async (response) => {
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new ApiRequestError(response.status)
|
throw new ApiRequestError(response.status)
|
||||||
}
|
}
|
||||||
const session = (await response.json()) as SessionResponse
|
const session = (await response.json()) as SessionResponse
|
||||||
|
cachedSessionToken = session.token
|
||||||
return session.token
|
return session.token
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
pendingSessionToken = null
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return pendingSessionToken
|
||||||
}
|
}
|
||||||
|
|
||||||
function withQuery(path: string, values: object): string {
|
function withQuery(path: string, values: object): string {
|
||||||
@@ -265,3 +293,13 @@ export function listDirectories(
|
|||||||
{ signal },
|
{ signal },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 仪表盘计数:一次请求拿到全部指标。 */
|
||||||
|
export function fetchStats(signal?: AbortSignal): Promise<TaskStats> {
|
||||||
|
return requestJson<TaskStats>('/api/v1/stats', { signal })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 已启用任务的下次触发时间。 */
|
||||||
|
export function fetchSchedule(signal?: AbortSignal): Promise<ScheduledTask[]> {
|
||||||
|
return requestJson<ScheduledTask[]>('/api/v1/schedule', { signal })
|
||||||
|
}
|
||||||
|
|||||||
@@ -132,6 +132,22 @@ export interface paths {
|
|||||||
patch?: never;
|
patch?: never;
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/api/v1/schedule": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: operations["list_schedule"];
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/api/v1/session": {
|
"/api/v1/session": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
@@ -148,6 +164,22 @@ export interface paths {
|
|||||||
patch?: never;
|
patch?: never;
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/api/v1/stats": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
get: operations["task_stats"];
|
||||||
|
put?: never;
|
||||||
|
post?: never;
|
||||||
|
delete?: never;
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
patch?: never;
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/api/v1/system/directories": {
|
"/api/v1/system/directories": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
@@ -338,6 +370,15 @@ export interface components {
|
|||||||
RunStatus: "queued" | "running" | "succeeded" | "failed" | "timed_out" | "cancelled" | "skipped";
|
RunStatus: "queued" | "running" | "succeeded" | "failed" | "timed_out" | "cancelled" | "skipped";
|
||||||
/** @enum {string} */
|
/** @enum {string} */
|
||||||
RunTrigger: "schedule" | "manual" | "retry";
|
RunTrigger: "schedule" | "manual" | "retry";
|
||||||
|
/** @description 已注册调度的任务及其下次触发时间。 */
|
||||||
|
ScheduledTaskResponse: {
|
||||||
|
/**
|
||||||
|
* Format: date-time
|
||||||
|
* @description 下次触发时间;调度正在重建时可能为空
|
||||||
|
*/
|
||||||
|
next_run_at?: string | null;
|
||||||
|
task_id: string;
|
||||||
|
};
|
||||||
SessionResponse: {
|
SessionResponse: {
|
||||||
/** @description Random per-process token required for mutating browser requests. */
|
/** @description Random per-process token required for mutating browser requests. */
|
||||||
token: string;
|
token: string;
|
||||||
@@ -427,6 +468,39 @@ export interface components {
|
|||||||
task_id: string;
|
task_id: string;
|
||||||
trigger: components["schemas"]["RunTrigger"];
|
trigger: components["schemas"]["RunTrigger"];
|
||||||
};
|
};
|
||||||
|
/** @description 仪表盘指标:一次请求拿到全部计数,避免为每个指标各发一次列表查询。 */
|
||||||
|
TaskStatsResponse: {
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description 已启用任务数
|
||||||
|
*/
|
||||||
|
enabled: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description HTTP 类任务数
|
||||||
|
*/
|
||||||
|
http: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description 进程类任务数
|
||||||
|
*/
|
||||||
|
process: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description 当前正在执行的任务数
|
||||||
|
*/
|
||||||
|
running: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description 脚本类任务数
|
||||||
|
*/
|
||||||
|
script: number;
|
||||||
|
/**
|
||||||
|
* Format: int64
|
||||||
|
* @description 任务总数
|
||||||
|
*/
|
||||||
|
total: number;
|
||||||
|
};
|
||||||
UpdateTaskRequest: {
|
UpdateTaskRequest: {
|
||||||
action: components["schemas"]["TaskAction"];
|
action: components["schemas"]["TaskAction"];
|
||||||
concurrency_policy?: components["schemas"]["ConcurrencyPolicy"];
|
concurrency_policy?: components["schemas"]["ConcurrencyPolicy"];
|
||||||
@@ -718,6 +792,26 @@ export interface operations {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
list_schedule: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Next trigger time per enabled task */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["ScheduledTaskResponse"][];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
session: {
|
session: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
@@ -738,6 +832,26 @@ export interface operations {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
task_stats: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Dashboard counters */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["TaskStatsResponse"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
list_directories: {
|
list_directories: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: {
|
query?: {
|
||||||
|
|||||||
@@ -17,6 +17,21 @@ export function formatDateTime(value: string | null | undefined): string {
|
|||||||
return Number.isNaN(date.getTime()) ? '--' : dateTimeFormatter.format(date)
|
return Number.isNaN(date.getTime()) ? '--' : dateTimeFormatter.format(date)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const shortDateTimeFormatter = new Intl.DateTimeFormat('zh-CN', {
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit',
|
||||||
|
})
|
||||||
|
|
||||||
|
export function formatShortDateTime(value: string | null | undefined): string {
|
||||||
|
if (!value) {
|
||||||
|
return '--'
|
||||||
|
}
|
||||||
|
const date = new Date(value)
|
||||||
|
return Number.isNaN(date.getTime()) ? '--' : shortDateTimeFormatter.format(date)
|
||||||
|
}
|
||||||
|
|
||||||
export function formatDuration(value: number | null | undefined): string {
|
export function formatDuration(value: number | null | undefined): string {
|
||||||
if (value === null || value === undefined) {
|
if (value === null || value === undefined) {
|
||||||
return '--'
|
return '--'
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import {
|
|||||||
createTask,
|
createTask,
|
||||||
deleteTask,
|
deleteTask,
|
||||||
fetchHealth,
|
fetchHealth,
|
||||||
|
fetchSchedule,
|
||||||
|
fetchStats,
|
||||||
listTasks,
|
listTasks,
|
||||||
runTask,
|
runTask,
|
||||||
setTaskEnabled,
|
setTaskEnabled,
|
||||||
@@ -24,7 +26,7 @@ import {
|
|||||||
EnabledBadge,
|
EnabledBadge,
|
||||||
} from '../components/StatusBadge'
|
} from '../components/StatusBadge'
|
||||||
import { TaskEditorModal } from '../components/TaskEditorModal'
|
import { TaskEditorModal } from '../components/TaskEditorModal'
|
||||||
import { actionTypeLabel } from '../lib/format'
|
import { actionTypeLabel, formatShortDateTime } from '../lib/format'
|
||||||
import {
|
import {
|
||||||
type TaskFormValues,
|
type TaskFormValues,
|
||||||
valuesToCreateRequest,
|
valuesToCreateRequest,
|
||||||
@@ -75,23 +77,19 @@ export function TaskDashboard() {
|
|||||||
queryFn: ({ signal }) => listTasks(filters, signal),
|
queryFn: ({ signal }) => listTasks(filters, signal),
|
||||||
placeholderData: keepPreviousData,
|
placeholderData: keepPreviousData,
|
||||||
})
|
})
|
||||||
const totalTasksQuery = useQuery({
|
const statsQuery = useQuery({
|
||||||
queryKey: ['tasks', 'summary', 'total'],
|
queryKey: ['stats'],
|
||||||
queryFn: ({ signal }) => listTasks({ page: 1, page_size: 1 }, signal),
|
queryFn: ({ signal }) => fetchStats(signal),
|
||||||
|
refetchInterval: 15_000,
|
||||||
})
|
})
|
||||||
const enabledTasksQuery = useQuery({
|
const scheduleQuery = useQuery({
|
||||||
queryKey: ['tasks', 'summary', 'enabled'],
|
queryKey: ['schedule'],
|
||||||
queryFn: ({ signal }) =>
|
queryFn: ({ signal }) => fetchSchedule(signal),
|
||||||
listTasks({ enabled: true, page: 1, page_size: 1 }, signal),
|
refetchInterval: 15_000,
|
||||||
})
|
|
||||||
const scriptTasksQuery = useQuery({
|
|
||||||
queryKey: ['tasks', 'summary', 'script'],
|
|
||||||
queryFn: ({ signal }) =>
|
|
||||||
listTasks(
|
|
||||||
{ action_type: 'script', page: 1, page_size: 1 },
|
|
||||||
signal,
|
|
||||||
),
|
|
||||||
})
|
})
|
||||||
|
const nextRuns = new Map(
|
||||||
|
(scheduleQuery.data ?? []).map((entry) => [entry.task_id, entry.next_run_at]),
|
||||||
|
)
|
||||||
|
|
||||||
const saveMutation = useMutation({
|
const saveMutation = useMutation({
|
||||||
mutationFn: ({
|
mutationFn: ({
|
||||||
@@ -232,18 +230,18 @@ export function TaskDashboard() {
|
|||||||
<section className="metrics-grid" aria-label="任务统计">
|
<section className="metrics-grid" aria-label="任务统计">
|
||||||
<MetricCard
|
<MetricCard
|
||||||
label="已托管任务"
|
label="已托管任务"
|
||||||
value={totalTasksQuery.data?.total}
|
value={statsQuery.data?.total}
|
||||||
loading={totalTasksQuery.isPending}
|
loading={statsQuery.isPending}
|
||||||
/>
|
/>
|
||||||
<MetricCard
|
<MetricCard
|
||||||
label="正常运行中"
|
label="运行中任务"
|
||||||
value={enabledTasksQuery.data?.total}
|
value={statsQuery.data?.running}
|
||||||
loading={enabledTasksQuery.isPending}
|
loading={statsQuery.isPending}
|
||||||
/>
|
/>
|
||||||
<MetricCard
|
<MetricCard
|
||||||
label="可用脚本"
|
label="可用脚本"
|
||||||
value={scriptTasksQuery.data?.total}
|
value={statsQuery.data?.script}
|
||||||
loading={scriptTasksQuery.isPending}
|
loading={statsQuery.isPending}
|
||||||
/>
|
/>
|
||||||
<article className="metric-card">
|
<article className="metric-card">
|
||||||
<span>Rust 引擎</span>
|
<span>Rust 引擎</span>
|
||||||
@@ -407,6 +405,13 @@ export function TaskDashboard() {
|
|||||||
<td>
|
<td>
|
||||||
<code>{task.cron}</code>
|
<code>{task.cron}</code>
|
||||||
<div className="cell-note">{task.timezone}</div>
|
<div className="cell-note">{task.timezone}</div>
|
||||||
|
<div className="cell-note">
|
||||||
|
{task.enabled
|
||||||
|
? nextRuns.get(task.id)
|
||||||
|
? `下次 ${formatShortDateTime(nextRuns.get(task.id))}`
|
||||||
|
: '调度中…'
|
||||||
|
: '未调度'}
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<span className="type-badge">
|
<span className="type-badge">
|
||||||
@@ -573,5 +578,7 @@ async function invalidateTaskData(queryClient: ReturnType<typeof useQueryClient>
|
|||||||
await Promise.all([
|
await Promise.all([
|
||||||
queryClient.invalidateQueries({ queryKey: ['tasks'] }),
|
queryClient.invalidateQueries({ queryKey: ['tasks'] }),
|
||||||
queryClient.invalidateQueries({ queryKey: ['task-runs'] }),
|
queryClient.invalidateQueries({ queryKey: ['task-runs'] }),
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['stats'] }),
|
||||||
|
queryClient.invalidateQueries({ queryKey: ['schedule'] }),
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
$projectRoot = Split-Path -Parent $PSScriptRoot
|
$projectRoot = Split-Path -Parent $PSScriptRoot
|
||||||
|
$scratch = Join-Path $projectRoot '.tmp'
|
||||||
|
|
||||||
function Assert-ExitCode {
|
function Assert-ExitCode {
|
||||||
param([string]$Step)
|
param([string]$Step)
|
||||||
@@ -10,6 +11,12 @@ function Assert-ExitCode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# 测试统一使用仓库内的临时目录:不依赖系统 %TEMP% 的权限策略,
|
||||||
|
# 也让清理、归档和排障都留在项目目录里。
|
||||||
|
[void](New-Item -ItemType Directory -Path $scratch -Force)
|
||||||
|
$env:TMP = $scratch
|
||||||
|
$env:TEMP = $scratch
|
||||||
|
|
||||||
Push-Location $projectRoot
|
Push-Location $projectRoot
|
||||||
try {
|
try {
|
||||||
pnpm --dir frontend lint
|
pnpm --dir frontend lint
|
||||||
@@ -31,4 +38,5 @@ try {
|
|||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
Pop-Location
|
Pop-Location
|
||||||
|
Remove-Item -LiteralPath $scratch -Recurse -Force -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user