feat: add runtime storage cleanup with per-category retention (#310) (#313)

Runtime artifacts (agent workspaces, tool-output spill, C2 payloads, chat
uploads, workflow checkpoints, diagnostic logs) previously accumulated
without bound: most were only removed when a conversation or project was
deleted, and tmp/c2 plus workflow checkpoints were never removed at all.

Add a storage cleaner with named per-category tasks (Gitea-style), a
settings page tab, and a background sweep that is off by default so
upgrading never deletes existing data.

Safety properties, since mis-deleting live task data costs far more than
the disk saved:
- dry-run is the default; a real cleanup requires dry_run=false together
  with confirm=true at the API layer, not just a frontend dialog
- sessions active within active_grace_hours are always skipped, and a
  failed activity lookup skips conservatively (fail closed)
- directories whose conversation/project no longer exists are reclaimed
  as orphans after orphan_grace_days
- scanners never follow symlinks and every candidate path is confined to
  its category root; deletion renames to a .tmp-for-deletion marker first
  so a crash leaves recoverable residue instead of a half-deleted dir
- storage:* permissions are admin-only; without the grantSystemRolePermissions
  skip the default branch would have given operators an irreversible
  file-deletion right

Also fix two confirmed leaks: DeleteConversation left chat_uploads files
on disk (their rows already vanished via ON DELETE CASCADE), and workflow
checkpoints had no deletion path at all.

Co-authored-by: Parallels <parallels@kali-linux-2025-2.localdomain>
This commit is contained in:
SycunandParallels authored and GitHub committed 2026-09-25 14:46:23 +08:00
1 parent e9b6e0d86e
commit 470eb5ead1
28 files changed
+3356 -1

No files matched your search

@@ -60,6 +60,67 @@ func TestDeleteConversationRemovesEinoScopedDirs(t *testing.T) {
}
}
// chat_uploads 比其他产物多一层日期目录(chat_uploads/<日期>/<会话ID>),
// 历史上删除会话只清了数据库行(ON DELETE CASCADE),磁盘文件会永久残留。
func TestDeleteConversationRemovesChatUploads(t *testing.T) {
tmp := t.TempDir()
dbPath := filepath.Join(tmp, "conversations.db")
db, err := NewDB(dbPath, zap.NewNop())
if err != nil {
t.Fatalf("NewDB: %v", err)
}
defer db.Close()
uploads := filepath.Join(tmp, "chat_uploads")
db.SetChatUploadsDir(uploads)
target, err := db.CreateConversation("uploads cleanup", ConversationCreateMeta{})
if err != nil {
t.Fatalf("CreateConversation: %v", err)
}
sibling, err := db.CreateConversation("sibling", ConversationCreateMeta{})
if err != nil {
t.Fatalf("CreateConversation: %v", err)
}
targetSeg := sanitizeConversationPathSegment(target.ID)
siblingSeg := sanitizeConversationPathSegment(sibling.ID)
// 同一会话跨两个日期目录都有上传件,另一个会话的上传件必须保留。
for _, dir := range []string{
filepath.Join(uploads, "2026-06-01", targetSeg),
filepath.Join(uploads, "2026-06-02", targetSeg),
filepath.Join(uploads, "2026-06-01", siblingSeg),
} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("mkdir %s: %v", dir, err)
}
if err := os.WriteFile(filepath.Join(dir, "report.pdf"), []byte("x"), 0o644); err != nil {
t.Fatalf("write %s: %v", dir, err)
}
}
if err := db.DeleteConversation(target.ID); err != nil {
t.Fatalf("DeleteConversation: %v", err)
}
for _, date := range []string{"2026-06-01", "2026-06-02"} {
dir := filepath.Join(uploads, date, targetSeg)
if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) {
t.Errorf("expected removed dir %s, stat err=%v", dir, statErr)
}
}
siblingDir := filepath.Join(uploads, "2026-06-01", siblingSeg)
if _, statErr := os.Stat(filepath.Join(siblingDir, "report.pdf")); statErr != nil {
t.Errorf("其他会话的上传件被误删: %v", statErr)
}
if _, statErr := os.Stat(uploads); statErr != nil {
t.Errorf("chat_uploads 根目录不应被删除: %v", statErr)
}
if _, statErr := os.Stat(filepath.Join(uploads, "2026-06-01")); statErr != nil {
t.Errorf("仍有数据的日期目录不应被删除: %v", statErr)
}
}
func TestDeleteProjectRemovesReductionDir(t *testing.T) {
tmp := t.TempDir()
dbPath := filepath.Join(tmp, "conversations.db")