feat: add runtime storage cleanup with per-category retention (#310) (#313)

Runtime artifacts (agent workspaces, tool-output spill, C2 payloads, chat
uploads, workflow checkpoints, diagnostic logs) previously accumulated
without bound: most were only removed when a conversation or project was
deleted, and tmp/c2 plus workflow checkpoints were never removed at all.

Add a storage cleaner with named per-category tasks (Gitea-style), a
settings page tab, and a background sweep that is off by default so
upgrading never deletes existing data.

Safety properties, since mis-deleting live task data costs far more than
the disk saved:
- dry-run is the default; a real cleanup requires dry_run=false together
  with confirm=true at the API layer, not just a frontend dialog
- sessions active within active_grace_hours are always skipped, and a
  failed activity lookup skips conservatively (fail closed)
- directories whose conversation/project no longer exists are reclaimed
  as orphans after orphan_grace_days
- scanners never follow symlinks and every candidate path is confined to
  its category root; deletion renames to a .tmp-for-deletion marker first
  so a crash leaves recoverable residue instead of a half-deleted dir
- storage:* permissions are admin-only; without the grantSystemRolePermissions
  skip the default branch would have given operators an irreversible
  file-deletion right

Also fix two confirmed leaks: DeleteConversation left chat_uploads files
on disk (their rows already vanished via ON DELETE CASCADE), and workflow
checkpoints had no deletion path at all.

Co-authored-by: Parallels <parallels@kali-linux-2025-2.localdomain>
This commit is contained in:
SycunandParallels authored and GitHub committed 2026-09-25 14:46:23 +08:00
1 parent e9b6e0d86e
commit 470eb5ead1
28 files changed
+3356 -1

No files matched your search

+65
View File
@@ -1784,6 +1784,7 @@
"security": "Security",
"rbac": "Platform permissions",
"audit": "Audit logs",
"storage": "Storage cleanup",
"infocollect": "Asset management"
},
"infocollect": {
@@ -3062,6 +3063,70 @@
"close": "×",
"newTerminal": "+"
},
"settingsStorage": {
"title": "Storage cleanup",
"subtitle": "Inspect and reclaim disk space used by runtime artifacts (agent workspaces, tool-output spill, C2 artifacts, chat uploads). Deletion is irreversible - preview first.",
"diskUsed": "Disk used",
"runtimeTotal": "Runtime footprint",
"reclaimable": "Reclaimable",
"inodes": "inode usage",
"inodesHint": "Scans can exhaust inodes with huge numbers of tiny files",
"refresh": "Rescan",
"preview": "Preview reclaimable",
"cleanNow": "Clean now",
"colCategory": "Category",
"colRoot": "Directory",
"colUnits": "Items",
"colSize": "Size",
"colReclaimable": "Reclaimable",
"colRetention": "Retention days",
"colEnabled": "Enabled",
"loading": "Loading...",
"policyTitle": "Automatic cleanup policy",
"autoClean": "Enable scheduled background cleanup",
"intervalMinutes": "Interval (minutes)",
"orphanGraceDays": "Orphan grace (days)",
"activeGraceHours": "Active session protection (hours)",
"policyHint": "Retention of 0 disables age-based cleanup, but directories whose conversation is already gone are still reclaimed. Sessions with recent activity are never cleaned. Automatic cleanup is off by default, so upgrading never deletes existing data.",
"save": "Save policy",
"loadMetaFailed": "Failed to load cleanup policy",
"loadStatusFailed": "Failed to load storage usage",
"cleanupFailed": "Cleanup request failed",
"unavailable": "Unavailable",
"free": "free",
"itemsUnit": "items",
"daysUnit": "days",
"noCategories": "No cleanable categories",
"orphanSuffix": "incl. orphans",
"skippedActive": "skipped active",
"defaultRetention": "Default",
"unused": "Unused",
"scannedAt": "Scanned at",
"previewTitle": "Preview (nothing was deleted)",
"cleanDone": "Cleanup finished",
"nothingToClean": "Nothing currently matches the cleanup policy.",
"skippedActiveNote": "Skipped sessions with recent activity: ",
"failed": "Operation failed",
"confirmClean": "This permanently deletes about {size} ({count} items) of runtime files and cannot be undone. Run \"Preview reclaimable\" first. Continue?",
"saveFailed": "Save failed",
"saved": "Cleanup policy saved",
"cat.workspace": "Agent workspace",
"catHint.workspace": "Agent download & scan artifacts (tmp/workspace), split per project/conversation.",
"cat.reduction": "Tool output spill",
"catHint.reduction": "Truncated spill files of oversized tool output (tmp/reduction); one file per execution, purely derived.",
"cat.conversation_artifacts": "Conversation artifacts",
"catHint.conversation_artifacts": "Summarization transcripts and the oversized user-input ledger (data/conversation_artifacts).",
"cat.plantask": "Plan task boards",
"catHint.plantask": "Eino multi-agent plan boards (skills/.eino/plantask), purely derived.",
"cat.c2_artifacts": "C2 artifacts",
"catHint.c2_artifacts": "C2 screenshots, uploads, pushed files and generated payloads (tmp/c2). Cleaning payloads invalidates their download links.",
"cat.chat_uploads": "Chat uploads",
"catHint.chat_uploads": "Files uploaded in conversations (chat_uploads/<date>/<conversation>).",
"cat.workflow_checkpoints": "Workflow checkpoints",
"catHint.workflow_checkpoints": "Workflow run checkpoints (data/workflow-checkpoints), only used to resume interrupted runs.",
"cat.diagnostic_logs": "Diagnostic logs",
"catHint.diagnostic_logs": "Daily-rotated diagnostic logs (log/diagnostic-*.log)."
},
"settingsAudit": {
"title": "Audit logs",
"description": "Platform admin actions (login, config, deletes). Does not log chat content, per-command terminal/WebShell runs, or per-tool invocations.",
+65
View File
@@ -1772,6 +1772,7 @@
"security": "安全设置",
"rbac": "平台权限",
"audit": "日志审计",
"storage": "存储清理",
"infocollect": "资产管理"
},
"infocollect": {
@@ -3050,6 +3051,70 @@
"close": "×",
"newTerminal": "+"
},
"settingsStorage": {
"title": "存储清理",
"subtitle": "统计并清理运行期间产生的磁盘垃圾(Agent 工作区、工具输出缓存、C2 产物、对话上传件等)。删除不可逆,建议先预览再执行。",
"diskUsed": "磁盘已用",
"runtimeTotal": "运行空间占用",
"reclaimable": "可回收",
"inodes": "inode 使用",
"inodesHint": "扫描产生的海量小文件可能先耗尽 inode",
"refresh": "重新扫描",
"preview": "预览可清理项",
"cleanNow": "立即清理",
"colCategory": "类别",
"colRoot": "目录",
"colUnits": "项目数",
"colSize": "占用",
"colReclaimable": "可回收",
"colRetention": "保留天数",
"colEnabled": "启用",
"loading": "加载中…",
"policyTitle": "自动清理策略",
"autoClean": "开启后台定时清理",
"intervalMinutes": "清理间隔(分钟)",
"orphanGraceDays": "孤儿目录宽限(天)",
"activeGraceHours": "活跃会话保护(小时)",
"policyHint": "保留天数为 0 表示不按保留期清理,但会话已删除的孤儿目录仍会回收;最近有活动的会话在任何情况下都不会被清理。默认关闭自动清理,升级不会自动删除既有数据。",
"save": "保存策略",
"loadMetaFailed": "获取清理策略失败",
"loadStatusFailed": "获取存储占用失败",
"cleanupFailed": "清理请求失败",
"unavailable": "不可用",
"free": "可用",
"itemsUnit": "项",
"daysUnit": "天",
"noCategories": "暂无可清理类别",
"orphanSuffix": "含孤儿",
"skippedActive": "跳过活跃",
"defaultRetention": "默认",
"unused": "未使用",
"scannedAt": "扫描于",
"previewTitle": "预览结果(未删除任何文件)",
"cleanDone": "清理完成",
"nothingToClean": "当前没有符合清理条件的内容。",
"skippedActiveNote": "已跳过最近仍在活动的会话:",
"failed": "操作失败",
"confirmClean": "将永久删除约 {size}({count} 项)运行空间文件,无法恢复。建议先执行「预览可清理项」。确认继续?",
"saveFailed": "保存失败",
"saved": "清理策略已保存",
"cat.workspace": "Agent 工作区",
"catHint.workspace": "Agent 下载与扫描产物的工作目录(tmp/workspace),按项目/会话分目录。",
"cat.reduction": "工具输出缓存",
"catHint.reduction": "超长工具输出落盘的截断文件(tmp/reduction),每条执行一个文件,属纯派生数据。",
"cat.conversation_artifacts": "会话产物",
"catHint.conversation_artifacts": "摘要记录与超长用户输入台账(data/conversation_artifacts)。",
"cat.plantask": "计划任务看板",
"catHint.plantask": "Eino 多代理计划看板 JSON(skills/.eino/plantask),属纯派生数据。",
"cat.c2_artifacts": "C2 产物",
"catHint.c2_artifacts": "C2 回传截图、上传件、下发文件与已生成的 payload 二进制(tmp/c2)。清理后对应 payload 下载链接会失效。",
"cat.chat_uploads": "对话上传文件",
"catHint.chat_uploads": "用户在对话中上传的附件(chat_uploads/日期/会话)。",
"cat.workflow_checkpoints": "工作流检查点",
"catHint.workflow_checkpoints": "工作流运行断点文件(data/workflow-checkpoints),仅用于恢复中断的运行。",
"cat.diagnostic_logs": "诊断日志",
"catHint.diagnostic_logs": "按天轮转的诊断日志(log/diagnostic-*.log)。"
},
"settingsAudit": {
"title": "日志审计",
"description": "记录平台管理类操作(登录、配置、删除等),不记录对话正文、终端/WebShell 每次命令与工具调用明细。",