mirror of
https://github.com/Ed1s0nZ/CyberStrikeAI.git
synced 2026-10-01 13:40:36 +02:00
Runtime artifacts (agent workspaces, tool-output spill, C2 payloads, chat uploads, workflow checkpoints, diagnostic logs) previously accumulated without bound: most were only removed when a conversation or project was deleted, and tmp/c2 plus workflow checkpoints were never removed at all. Add a storage cleaner with named per-category tasks (Gitea-style), a settings page tab, and a background sweep that is off by default so upgrading never deletes existing data. Safety properties, since mis-deleting live task data costs far more than the disk saved: - dry-run is the default; a real cleanup requires dry_run=false together with confirm=true at the API layer, not just a frontend dialog - sessions active within active_grace_hours are always skipped, and a failed activity lookup skips conservatively (fail closed) - directories whose conversation/project no longer exists are reclaimed as orphans after orphan_grace_days - scanners never follow symlinks and every candidate path is confined to its category root; deletion renames to a .tmp-for-deletion marker first so a crash leaves recoverable residue instead of a half-deleted dir - storage:* permissions are admin-only; without the grantSystemRolePermissions skip the default branch would have given operators an irreversible file-deletion right Also fix two confirmed leaks: DeleteConversation left chat_uploads files on disk (their rows already vanished via ON DELETE CASCADE), and workflow checkpoints had no deletion path at all. Co-authored-by: Parallels <parallels@kali-linux-2025-2.localdomain>
170 lines
5.8 KiB
Go
170 lines
5.8 KiB
Go
package database
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
func TestDeleteConversationRemovesEinoScopedDirs(t *testing.T) {
|
|
tmp := t.TempDir()
|
|
dbPath := filepath.Join(tmp, "conversations.db")
|
|
db, err := NewDB(dbPath, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("NewDB: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
plantaskBase := filepath.Join(tmp, "skills", ".eino", "plantask")
|
|
checkpointBase := filepath.Join(tmp, "eino-checkpoints")
|
|
reductionBase := filepath.Join(tmp, "reduction")
|
|
workspaceBase := filepath.Join(tmp, "workspace")
|
|
db.SetEinoConversationDirs(plantaskBase, checkpointBase, reductionBase, workspaceBase)
|
|
|
|
conv, err := db.CreateConversation("cleanup test", ConversationCreateMeta{})
|
|
if err != nil {
|
|
t.Fatalf("CreateConversation: %v", err)
|
|
}
|
|
convID := conv.ID
|
|
seg := sanitizeConversationPathSegment(convID)
|
|
for _, base := range []struct {
|
|
root string
|
|
file string
|
|
}{
|
|
{db.conversationArtifactsDir, "transcript.txt"},
|
|
{plantaskBase, "task-1.json"},
|
|
{checkpointBase, "runner-deep.ckpt"},
|
|
{filepath.Join(reductionBase, "conversations"), "tool-output.txt"},
|
|
{filepath.Join(workspaceBase, "conversations"), "page.html"},
|
|
} {
|
|
dir := filepath.Join(base.root, seg)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", dir, err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, base.file), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("write %s: %v", base.file, err)
|
|
}
|
|
}
|
|
|
|
if err := db.DeleteConversation(convID); err != nil {
|
|
t.Fatalf("DeleteConversation: %v", err)
|
|
}
|
|
|
|
for _, base := range []string{db.conversationArtifactsDir, plantaskBase, checkpointBase, filepath.Join(reductionBase, "conversations"), filepath.Join(workspaceBase, "conversations")} {
|
|
dir := filepath.Join(base, seg)
|
|
if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) {
|
|
t.Fatalf("expected removed dir %s, stat err=%v", dir, statErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
// chat_uploads 比其他产物多一层日期目录(chat_uploads/<日期>/<会话ID>),
|
|
// 历史上删除会话只清了数据库行(ON DELETE CASCADE),磁盘文件会永久残留。
|
|
func TestDeleteConversationRemovesChatUploads(t *testing.T) {
|
|
tmp := t.TempDir()
|
|
dbPath := filepath.Join(tmp, "conversations.db")
|
|
db, err := NewDB(dbPath, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("NewDB: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
uploads := filepath.Join(tmp, "chat_uploads")
|
|
db.SetChatUploadsDir(uploads)
|
|
|
|
target, err := db.CreateConversation("uploads cleanup", ConversationCreateMeta{})
|
|
if err != nil {
|
|
t.Fatalf("CreateConversation: %v", err)
|
|
}
|
|
sibling, err := db.CreateConversation("sibling", ConversationCreateMeta{})
|
|
if err != nil {
|
|
t.Fatalf("CreateConversation: %v", err)
|
|
}
|
|
targetSeg := sanitizeConversationPathSegment(target.ID)
|
|
siblingSeg := sanitizeConversationPathSegment(sibling.ID)
|
|
|
|
// 同一会话跨两个日期目录都有上传件,另一个会话的上传件必须保留。
|
|
for _, dir := range []string{
|
|
filepath.Join(uploads, "2026-06-01", targetSeg),
|
|
filepath.Join(uploads, "2026-06-02", targetSeg),
|
|
filepath.Join(uploads, "2026-06-01", siblingSeg),
|
|
} {
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", dir, err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "report.pdf"), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("write %s: %v", dir, err)
|
|
}
|
|
}
|
|
|
|
if err := db.DeleteConversation(target.ID); err != nil {
|
|
t.Fatalf("DeleteConversation: %v", err)
|
|
}
|
|
|
|
for _, date := range []string{"2026-06-01", "2026-06-02"} {
|
|
dir := filepath.Join(uploads, date, targetSeg)
|
|
if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) {
|
|
t.Errorf("expected removed dir %s, stat err=%v", dir, statErr)
|
|
}
|
|
}
|
|
siblingDir := filepath.Join(uploads, "2026-06-01", siblingSeg)
|
|
if _, statErr := os.Stat(filepath.Join(siblingDir, "report.pdf")); statErr != nil {
|
|
t.Errorf("其他会话的上传件被误删: %v", statErr)
|
|
}
|
|
if _, statErr := os.Stat(uploads); statErr != nil {
|
|
t.Errorf("chat_uploads 根目录不应被删除: %v", statErr)
|
|
}
|
|
if _, statErr := os.Stat(filepath.Join(uploads, "2026-06-01")); statErr != nil {
|
|
t.Errorf("仍有数据的日期目录不应被删除: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestDeleteProjectRemovesReductionDir(t *testing.T) {
|
|
tmp := t.TempDir()
|
|
dbPath := filepath.Join(tmp, "conversations.db")
|
|
db, err := NewDB(dbPath, zap.NewNop())
|
|
if err != nil {
|
|
t.Fatalf("NewDB: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
reductionBase := filepath.Join(tmp, "reduction")
|
|
workspaceBase := filepath.Join(tmp, "workspace")
|
|
db.SetEinoConversationDirs("", "", reductionBase, workspaceBase)
|
|
|
|
project, err := db.CreateProject(&Project{Name: "cleanup test"})
|
|
if err != nil {
|
|
t.Fatalf("CreateProject: %v", err)
|
|
}
|
|
seg := sanitizeConversationPathSegment(project.ID)
|
|
reductionDir := filepath.Join(reductionBase, "projects", seg, "clear")
|
|
if err := os.MkdirAll(reductionDir, 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", reductionDir, err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(reductionDir, "call-1.txt"), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("write: %v", err)
|
|
}
|
|
workspaceDir := filepath.Join(workspaceBase, "projects", seg, "downloads")
|
|
if err := os.MkdirAll(workspaceDir, 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", workspaceDir, err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(workspaceDir, "app.js"), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("write workspace: %v", err)
|
|
}
|
|
|
|
if err := db.DeleteProject(project.ID); err != nil {
|
|
t.Fatalf("DeleteProject: %v", err)
|
|
}
|
|
|
|
projectReductionDir := filepath.Join(reductionBase, "projects", seg)
|
|
if _, statErr := os.Stat(projectReductionDir); !os.IsNotExist(statErr) {
|
|
t.Fatalf("expected removed dir %s, stat err=%v", projectReductionDir, statErr)
|
|
}
|
|
projectWorkspaceDir := filepath.Join(workspaceBase, "projects", seg)
|
|
if _, statErr := os.Stat(projectWorkspaceDir); !os.IsNotExist(statErr) {
|
|
t.Fatalf("expected removed dir %s, stat err=%v", projectWorkspaceDir, statErr)
|
|
}
|
|
}
|