feat: 加固运行期可靠性并补齐调度与观测接口

- 运行句柄按 run_id 归属，修复取消后重跑时闸门被旧运行删除的竞态
- 调度循环改用取消令牌退出，启动时对账遗留运行，校验 rows_affected
- 三类任务统一参与调度，新增 /api/v1/stats 与 /api/v1/schedule
- SQLite 启用 WAL 与 busy_timeout，日志清理跳过活跃文件，单行上限 256 KiB
- 前端缓存管理令牌并在 403 时重试一次，仪表盘改用统计接口

backend/src/runs/executor.rs

@@ -16,7 +16,7 @@ use reqwest::Method;
 use serde::Serialize;
 use sqlx::SqlitePool;
 use tokio::{
-    io::{AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader},
+    io::{AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncWriteExt, BufReader},
     process::{Child, Command},
     sync::{Mutex as AsyncMutex, broadcast},
     time::{Duration, sleep, timeout},
@@ -35,6 +35,7 @@ use crate::{
 };
 
 const MAX_LOG_BYTES: u64 = 5 * 1024 * 1024;
+const MAX_LOG_LINE_BYTES: usize = 256 * 1024;
 const MAX_HTTP_LOG_BODY_BYTES: usize = 64 * 1024;
 const TERMINATION_WAIT_SECS: u64 = 5;
 const CREATE_NO_WINDOW: u32 = 0x0800_0000;
@@ -255,7 +256,7 @@ impl TaskExecutor {
         let run = match repository::create_queued(&self.inner.pool, &task.id, trigger).await {
             Ok(run) => run,
             Err(error) => {
-                lock_running(&self.inner.running).remove(&task.id);
+                remove_running_if(&self.inner.running, &task.id, None);
                 return Err(error);
             }
         };
@@ -301,6 +302,11 @@ impl TaskExecutor {
         self.inner.events.subscribe()
     }
 
+    /// 当前正在执行的任务数量（用于统计接口）。
+    pub fn running_count(&self) -> usize {
+        lock_running(&self.inner.running).len()
+    }
+
     pub fn log_path(&self, task_id: &str, run_id: i64) -> PathBuf {
         self.inner
             .logs_dir
@@ -320,18 +326,44 @@ impl TaskExecutor {
         let _guard = self.inner.cleanup_lock.lock().await;
         let logs_dir = self.inner.logs_dir.clone();
         let config = self.inner.log_config.clone();
-        tokio::task::spawn_blocking(move || cleanup_logs_sync(&logs_dir, &config))
+        let active = self.active_log_paths();
+        tokio::task::spawn_blocking(move || cleanup_logs_sync(&logs_dir, &config, &active))
             .await
             .map_err(std::io::Error::other)?
     }
 
+    fn active_log_paths(&self) -> std::collections::HashSet<PathBuf> {
+        lock_running(&self.inner.running)
+            .iter()
+            .filter_map(|(task_id, handle)| {
+                handle.run_id.map(|run_id| {
+                    self.inner
+                        .logs_dir
+                        .join(task_id)
+                        .join(format!("{run_id}.log"))
+                })
+            })
+            .collect()
+    }
+
     async fn execute_run(&self, task: TaskResponse, run_id: i64, cancellation: CancellationToken) {
         let started_at = Utc::now();
         let started = Instant::now();
-        if let Err(error) = repository::mark_running(&self.inner.pool, run_id, started_at).await {
-            tracing::error!(run_id, error = %error, "failed to mark task run as running");
-            lock_running(&self.inner.running).remove(&task.id);
-            return;
+        match repository::mark_running(&self.inner.pool, run_id, started_at).await {
+            Ok(true) => {}
+            Ok(false) => {
+                tracing::warn!(
+                    run_id,
+                    "run row disappeared before it could be marked running"
+                );
+                remove_running_if(&self.inner.running, &task.id, Some(run_id));
+                return;
+            }
+            Err(error) => {
+                tracing::error!(run_id, error = %error, "failed to mark task run as running");
+                remove_running_if(&self.inner.running, &task.id, Some(run_id));
+                return;
+            }
         }
         self.emit_run_by_id(run_id).await;
 
@@ -339,7 +371,7 @@ impl TaskExecutor {
         let finished_at = Utc::now();
         let duration_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
 
-        if let Err(error) = repository::complete(
+        match repository::complete(
             &self.inner.pool,
             run_id,
             outcome.status,
@@ -350,11 +382,19 @@ impl TaskExecutor {
         )
         .await
         {
-            tracing::error!(run_id, error = %error, "failed to complete task run");
+            Ok(true) => {}
+            Ok(false) => {
+                tracing::warn!(
+                    run_id,
+                    "run row disappeared before completion could be recorded"
+                );
+            }
+            Err(error) => tracing::error!(run_id, error = %error, "failed to complete task run"),
         }
 
         self.emit_run_by_id(run_id).await;
-        lock_running(&self.inner.running).remove(&task.id);
+        // 只清理属于自己的句柄：取消后立刻重跑时，旧 run 收尾不能删掉新 run 的闸门。
+        remove_running_if(&self.inner.running, &task.id, Some(run_id));
         if let Err(error) = self.cleanup_logs().await {
             tracing::warn!(%error, "failed to clean task logs");
         }
@@ -852,6 +892,21 @@ fn lock_running(
         .unwrap_or_else(|poisoned| poisoned.into_inner())
 }
 
+/// 只在句柄仍属于指定 run 时移除；`None` 表示"刚插入、还没拿到 run id"的占位句柄。
+fn remove_running_if(
+    running: &Mutex<HashMap<String, RunningHandle>>,
+    task_id: &str,
+    run_id: Option<i64>,
+) {
+    let mut running = lock_running(running);
+    if running
+        .get(task_id)
+        .is_some_and(|handle| handle.run_id == run_id)
+    {
+        running.remove(task_id);
+    }
+}
+
 fn spawn_log_reader<R>(
     reader: R,
     stream: &'static str,
@@ -864,12 +919,17 @@ where
         let mut reader = BufReader::new(reader);
         let mut line = Vec::new();
         loop {
-            line.clear();
-            match reader.read_until(b'\n', &mut line).await {
-                Ok(0) => break,
-                Ok(_) => {
+            match read_line_capped(&mut reader, &mut line, MAX_LOG_LINE_BYTES).await {
+                Ok((0, _)) => break,
+                Ok((_, truncated)) => {
                     let text = sink.decode_output(&line);
-                    sink.append(&format!("[{stream}] {text}")).await;
+                    let mut payload = format!("[{stream}] {text}");
+                    if truncated {
+                        payload.push_str(&format!(
+                            "\n[system] 单行输出超过 {MAX_LOG_LINE_BYTES} 字节，已截断\n"
+                        ));
+                    }
+                    sink.append(&payload).await;
                 }
                 Err(error) => {
                     tracing::warn!(stream, %error, "failed to read task output");
@@ -880,6 +940,38 @@ where
     })
 }
 
+/// 读取一行，最多保留 `max` 字节。超出上限时丢弃剩余部分但继续读到换行，
+/// 既保住内存上限，也不会让后续输出错位。
+async fn read_line_capped<R>(
+    reader: &mut R,
+    buffer: &mut Vec<u8>,
+    max: usize,
+) -> std::io::Result<(usize, bool)>
+where
+    R: AsyncBufRead + Unpin,
+{
+    buffer.clear();
+    let mut truncated = false;
+    loop {
+        let chunk = reader.fill_buf().await?;
+        if chunk.is_empty() {
+            return Ok((buffer.len(), truncated));
+        }
+        let newline = chunk.iter().position(|byte| *byte == b'\n');
+        let take = newline.map_or(chunk.len(), |index| index + 1);
+        let remaining = max.saturating_sub(buffer.len());
+        let copied = take.min(remaining);
+        buffer.extend_from_slice(&chunk[..copied]);
+        if copied < take {
+            truncated = true;
+        }
+        reader.consume(take);
+        if newline.is_some() {
+            return Ok((buffer.len(), truncated));
+        }
+    }
+}
+
 async fn wait_after_termination(child: &mut Child) {
     if timeout(Duration::from_secs(TERMINATION_WAIT_SECS), child.wait())
         .await
@@ -935,13 +1027,19 @@ fn failed_outcome(message: impl Into<String>) -> ScriptOutcome {
     }
 }
 
-fn cleanup_logs_sync(root: &Path, config: &LogRetentionConfig) -> std::io::Result<()> {
+fn cleanup_logs_sync(
+    root: &Path,
+    config: &LogRetentionConfig,
+    active: &std::collections::HashSet<PathBuf>,
+) -> std::io::Result<()> {
     if !root.exists() {
         return Ok(());
     }
 
     let mut files = Vec::new();
     collect_log_files(root, &mut files)?;
+    // 正在写入的日志不参与保留期/容量清理。
+    files.retain(|(path, _)| !active.contains(path));
     let retention = Duration::from_secs(config.retention_days.saturating_mul(86_400));
     let cutoff = SystemTime::now().checked_sub(retention);
 
@@ -1104,6 +1202,7 @@ mod tests {
                 retention_days: 30,
                 max_total_bytes: 10,
             },
+            &std::collections::HashSet::new(),
         )
         .unwrap();
         let total = std::fs::read_dir(temp.path())
@@ -1112,4 +1211,66 @@ mod tests {
             .sum::<u64>();
         assert!(total <= 10);
     }
+
+    #[test]
+    fn cleanup_skips_logs_of_running_tasks() {
+        let temp = tempfile::TempDir::new().unwrap();
+        let active = temp.path().join("active.log");
+        std::fs::write(&active, vec![b'x'; 32]).unwrap();
+        std::thread::sleep(std::time::Duration::from_millis(5));
+        let stale = temp.path().join("stale.log");
+        std::fs::write(&stale, vec![b'x'; 32]).unwrap();
+
+        let mut protected = std::collections::HashSet::new();
+        protected.insert(active.clone());
+        cleanup_logs_sync(
+            temp.path(),
+            &LogRetentionConfig {
+                retention_days: 0,
+                max_total_bytes: 1,
+            },
+            &protected,
+        )
+        .unwrap();
+
+        assert!(active.exists(), "active log must survive cleanup");
+        assert!(!stale.exists(), "stale log should be removed");
+    }
+
+    #[tokio::test]
+    async fn capped_line_reader_truncates_but_keeps_reading() {
+        let payload = [b'a'; 64];
+        let mut reader = BufReader::new(&payload[..]);
+        let mut buffer = Vec::new();
+
+        let (bytes, truncated) = read_line_capped(&mut reader, &mut buffer, 16)
+            .await
+            .unwrap();
+
+        assert_eq!(bytes, 16);
+        assert!(truncated);
+
+        let (next, next_truncated) = read_line_capped(&mut reader, &mut buffer, 16)
+            .await
+            .unwrap();
+        assert_eq!(next, 0);
+        assert!(!next_truncated);
+    }
+
+    #[test]
+    fn removes_only_the_matching_run_handle() {
+        let running = Mutex::new(HashMap::from([(
+            "task".to_owned(),
+            RunningHandle {
+                run_id: Some(2),
+                cancellation: CancellationToken::new(),
+            },
+        )]));
+
+        remove_running_if(&running, "task", Some(1));
+        assert!(lock_running(&running).contains_key("task"));
+
+        remove_running_if(&running, "task", Some(2));
+        assert!(!lock_running(&running).contains_key("task"));
+    }
 }