feat: add runtime storage cleanup with per-category retention (#310) (#313)

Runtime artifacts (agent workspaces, tool-output spill, C2 payloads, chat
uploads, workflow checkpoints, diagnostic logs) previously accumulated
without bound: most were only removed when a conversation or project was
deleted, and tmp/c2 plus workflow checkpoints were never removed at all.

Add a storage cleaner with named per-category tasks (Gitea-style), a
settings page tab, and a background sweep that is off by default so
upgrading never deletes existing data.

Safety properties, since mis-deleting live task data costs far more than
the disk saved:
- dry-run is the default; a real cleanup requires dry_run=false together
  with confirm=true at the API layer, not just a frontend dialog
- sessions active within active_grace_hours are always skipped, and a
  failed activity lookup skips conservatively (fail closed)
- directories whose conversation/project no longer exists are reclaimed
  as orphans after orphan_grace_days
- scanners never follow symlinks and every candidate path is confined to
  its category root; deletion renames to a .tmp-for-deletion marker first
  so a crash leaves recoverable residue instead of a half-deleted dir
- storage:* permissions are admin-only; without the grantSystemRolePermissions
  skip the default branch would have given operators an irreversible
  file-deletion right

Also fix two confirmed leaks: DeleteConversation left chat_uploads files
on disk (their rows already vanished via ON DELETE CASCADE), and workflow
checkpoints had no deletion path at all.

Co-authored-by: Parallels <parallels@kali-linux-2025-2.localdomain>
This commit is contained in:
SycunandParallels authored and GitHub committed 2026-09-25 14:46:23 +08:00
1 parent e9b6e0d86e
commit 470eb5ead1
28 files changed
+3356 -1

No files matched your search

+29
View File
@@ -835,6 +835,8 @@ func (db *DB) removeConversationScopedDirs(conversationID, projectID string) {
db.removeConversationScopedDir(db.einoPlantaskBaseDir, conversationID, "plantask")
// Eino ADK runner checkpoints (checkpoint_dir/<id>/).
db.removeConversationScopedDir(db.einoCheckpointBaseDir, conversationID, "eino_checkpoint")
// 上传附件始终归属单个会话,项目绑定的会话也要删,故放在 projectID 判断之外。
db.removeChatUploadDirs(conversationID)
// Eino reduction persisted tool outputs (tmp/reduction/conversations/<id>/).
// Project-bound sessions share projects/<id>/ — skip on single conversation delete.
if strings.TrimSpace(projectID) == "" {
@@ -845,6 +847,33 @@ func (db *DB) removeConversationScopedDirs(conversationID, projectID string) {
}
}
// removeChatUploadDirs 删除 chat_uploads/<日期>/<会话ID>/ 下属于该会话的上传目录。
// 该根目录比其他产物多一层日期目录,无法复用 removeConversationScopedDir。
func (db *DB) removeChatUploadDirs(conversationID string) {
base := strings.TrimSpace(db.chatUploadsDir)
if base == "" || strings.TrimSpace(conversationID) == "" {
return
}
seg := sanitizeConversationPathSegment(conversationID)
dates, err := os.ReadDir(base)
if err != nil {
return
}
for _, dateDir := range dates {
if !dateDir.IsDir() {
continue
}
dir := filepath.Join(base, dateDir.Name(), seg)
if rmErr := os.RemoveAll(dir); rmErr != nil && db.logger != nil {
db.logger.Warn("删除会话上传目录失败",
zap.String("conversationId", conversationID),
zap.String("kind", "chat_uploads"),
zap.String("dir", dir),
zap.Error(rmErr))
}
}
}
func (db *DB) removeProjectScopedDirs(projectID string) {
// Eino reduction persisted tool outputs (tmp/reduction/projects/<id>/).
reductionBase := filepath.Join(db.einoReductionBaseDir(), "projects")