feat: add runtime storage cleanup with per-category retention (#310) (#313)

Runtime artifacts (agent workspaces, tool-output spill, C2 payloads, chat
uploads, workflow checkpoints, diagnostic logs) previously accumulated
without bound: most were only removed when a conversation or project was
deleted, and tmp/c2 plus workflow checkpoints were never removed at all.

Add a storage cleaner with named per-category tasks (Gitea-style), a
settings page tab, and a background sweep that is off by default so
upgrading never deletes existing data.

Safety properties, since mis-deleting live task data costs far more than
the disk saved:
- dry-run is the default; a real cleanup requires dry_run=false together
  with confirm=true at the API layer, not just a frontend dialog
- sessions active within active_grace_hours are always skipped, and a
  failed activity lookup skips conservatively (fail closed)
- directories whose conversation/project no longer exists are reclaimed
  as orphans after orphan_grace_days
- scanners never follow symlinks and every candidate path is confined to
  its category root; deletion renames to a .tmp-for-deletion marker first
  so a crash leaves recoverable residue instead of a half-deleted dir
- storage:* permissions are admin-only; without the grantSystemRolePermissions
  skip the default branch would have given operators an irreversible
  file-deletion right

Also fix two confirmed leaks: DeleteConversation left chat_uploads files
on disk (their rows already vanished via ON DELETE CASCADE), and workflow
checkpoints had no deletion path at all.

Co-authored-by: Parallels <parallels@kali-linux-2025-2.localdomain>
This commit is contained in:
SycunandParallels authored and GitHub committed 2026-09-25 14:46:23 +08:00
1 parent e9b6e0d86e
commit 470eb5ead1
28 files changed
+3356 -1

No files matched your search

+80
View File
@@ -732,6 +732,7 @@ type UpdateConfigRequest struct {
Robots *config.RobotsConfig `json:"robots,omitempty"`
MultiAgent *config.MultiAgentAPIUpdate `json:"multi_agent,omitempty"`
C2 *config.C2APIUpdate `json:"c2,omitempty"`
Storage *config.StorageConfig `json:"storage,omitempty"`
}
// AgentConfigUpdate 用于 PATCH /api/config 的 agent 段:仅 JSON 中出现的字段(指针非 nil)覆盖内存配置。
@@ -919,6 +920,66 @@ func (h *ConfigHandler) UpdateConfig(c *gin.Context) {
)
}
if req.Storage != nil {
st := &h.config.Storage
if req.Storage.AutoClean != nil {
v := *req.Storage.AutoClean
st.AutoClean = &v
}
if req.Storage.IntervalMinutes != nil {
v := *req.Storage.IntervalMinutes
if v < 5 {
v = 5
}
st.IntervalMinutes = &v
}
if req.Storage.OrphanGraceDays != nil {
v := *req.Storage.OrphanGraceDays
if v < 0 {
v = 0
}
st.OrphanGraceDays = &v
}
if req.Storage.ActiveGraceHours != nil {
v := *req.Storage.ActiveGraceHours
if v < 1 {
v = 1
}
st.ActiveGraceHours = &v
}
if req.Storage.Categories != nil {
if st.Categories == nil {
st.Categories = make(map[string]config.StorageCategoryConfig, len(req.Storage.Categories))
}
// 只接受注册表内的类别键,未注册的键直接忽略,避免被写进 config.yaml。
for _, key := range config.StorageCategoryOrder {
patch, ok := req.Storage.Categories[key]
if !ok {
continue
}
cur := st.Categories[key]
if patch.Enabled != nil {
v := *patch.Enabled
cur.Enabled = &v
}
if patch.RetentionDays != nil {
v := *patch.RetentionDays
if v < 0 {
v = 0
}
cur.RetentionDays = &v
}
st.Categories[key] = cur
}
}
h.logger.Info("更新运行空间清理配置",
zap.Bool("auto_clean", st.AutoCleanEffective()),
zap.Int("interval_minutes", st.IntervalMinutesEffective()),
zap.Int("orphan_grace_days", st.OrphanGraceDaysEffective()),
zap.Int("active_grace_hours", st.ActiveGraceHoursEffective()),
)
}
// 更新Knowledge配置
if req.Knowledge != nil {
// 保存旧的嵌入模型配置(用于检测变更)
@@ -1838,6 +1899,7 @@ func (h *ConfigHandler) saveConfig() error {
updateC2Config(root, h.config.C2)
updateRobotsConfig(root, h.config.Robots)
updateHitlConfig(root, h.config.Hitl)
updateStorageConfig(root, h.config.Storage)
updateMultiAgentConfig(root, h.config.MultiAgent)
// 更新外部MCP配置(使用external_mcp.go中的函数,同一包中可直接调用)
updateExternalMCPConfig(root, h.config.ExternalMCP)
@@ -2223,6 +2285,24 @@ func updateHitlConfig(doc *yaml.Node, cfg config.HitlConfig) {
setStringInMap(hitlNode, "audit_agent_prompt_review_edit", cfg.AuditAgentPromptReviewEdit)
}
// updateStorageConfig 把运行空间清理策略写回 config.yaml,保留文件其余内容与注释。
func updateStorageConfig(doc *yaml.Node, cfg config.StorageConfig) {
root := doc.Content[0]
storageNode := ensureMap(root, "storage")
setBoolInMap(storageNode, "auto_clean", cfg.AutoCleanEffective())
setIntInMap(storageNode, "interval_minutes", cfg.IntervalMinutesEffective())
setIntInMap(storageNode, "orphan_grace_days", cfg.OrphanGraceDaysEffective())
setIntInMap(storageNode, "active_grace_hours", cfg.ActiveGraceHoursEffective())
// 按固定顺序输出,避免每次保存都因 map 迭代顺序不同而重排整个文件。
categoriesNode := ensureMap(storageNode, "categories")
for _, key := range config.StorageCategoryOrder {
categoryNode := ensureMap(categoriesNode, key)
setBoolInMap(categoryNode, "enabled", cfg.CategoryEnabled(key))
setIntInMap(categoryNode, "retention_days", cfg.CategoryRetentionDays(key))
}
}
// UpdateHitlDefaultConfig 更新全局默认人机协同配置并写入 config.yaml。
func (h *ConfigHandler) UpdateHitlDefaultConfig(mode, reviewer string, timeoutSeconds int) error {
h.mu.Lock()