From 68c01d342958a5206a20639c113d03b69fa3e9be Mon Sep 17 00:00:00 2001 From: Sycun <165354365+Sycun@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:43:07 +0800 Subject: [PATCH] feat: index compaction evidence for targeted context recovery --- docs/zh-CN/blackboard-context-memory.md | 18 +++ .../multiagent/eino_context_checkpoint.go | 59 ++++++- internal/multiagent/eino_evidence_index.go | 101 ++++++++++++ .../multiagent/eino_evidence_index_test.go | 144 ++++++++++++++++++ internal/projectprompt/blackboard.go | 2 +- 5 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 internal/multiagent/eino_evidence_index.go create mode 100644 internal/multiagent/eino_evidence_index_test.go diff --git a/docs/zh-CN/blackboard-context-memory.md b/docs/zh-CN/blackboard-context-memory.md index 4b0ca4d8..64df2c10 100644 --- a/docs/zh-CN/blackboard-context-memory.md +++ b/docs/zh-CN/blackboard-context-memory.md @@ -10,12 +10,28 @@ 压缩后的上下文会附上该记录的 `get_project_fact(key)` 入口，并刷新系统中的黑板索引。后续按需读取记录，再使用 `read_file` 分段查看原始证据；无须重新把完整历史注入每一次模型请求。 +新上下文和黑板记录都包含程序生成的证据索引：按实际写入文件的行号标出 Cookie / Set-Cookie、认证 / Token、请求 / 参数、错误 / 失败的关键词命中位置。索引只包含固定类别与位置，不复制凭据值；它是检索线索，不代表相关凭据仍有效或结论已验证。需要精确值而摘要缺失时，Agent 应先检查索引，使用 `grep` 定位，再用 `read_file` 分段核对附近记录的域名、请求和时间，不猜测原值。 + +每类在上下文中最多列出最近 3 个命中行号，完整索引文件每类最多列出最近 32 个；超出时提示总命中数与搜索入口。关键词索引不是完整语义索引，未命中也可以搜索快照原文。 + ## 原始证据不会随下一次压缩被覆盖 过去所有压缩轮次写入同一个 `transcript.txt`，后一次写入会覆盖前一次记录。现在每份压缩前历史按内容 SHA-256 保存到 `snapshots/transcript-.txt`，相同内容的重试复用快照，不重复建文件。`transcript.txt` 仍保留为最新记录的兼容入口，续接摘要引用的是具体快照。快照沿用现有会话记录格式，保留用户、助手与工具记录，省略不变的系统提示与工具/技能索引。 快照沿用 `conversation_artifacts//summarization/` 目录，与会话产物一起管理；子代理使用独立 scope 子目录。删除会话时现有清理逻辑会删除对应产物，快照不作为会话删除后的独立备份。文件通过临时文件原子替换写入，权限为 0600。 +默认数据库为 `data/conversations.db` 时，主代理文件布局如下（摘要向 Agent 提供绝对路径）： + +```text +data/conversation_artifacts//summarization/ +├── transcript.txt +└── snapshots/ + ├── transcript-.txt + └── transcript-.txt.index.md +``` + +子代理、规划器和执行器使用 `summarization/scopes//snapshots/`。每份快照有独立索引文件，相同内容重试复用同一文件；历史索引不随最新记录更新而消失。摘要提供历史快照目录，Agent 可用 `glob` 查找 `transcript-*.txt.index.md`，按需查找旧证据，无须维护多个代理并发覆盖的共享目录清单。 + 系统复用本次已经生成的压缩摘要，不增加一次额外的模型提取调用。原始历史在生成摘要前就做快照，因此即使摘要遗漏某个精确细节，也还有来源可以追溯。黑板或快照保存失败时会记录告警，保留当前可用摘要继续运行，不提示保存成功。 ## 配置和边界 @@ -36,6 +52,8 @@ project: 自动注入仍只包含 key、短摘要与关系边，完整 body 留在数据库按需读取。索引查询不再加载最终不会注入的 body。字符预算现在包含标题、起止标记、查询提示、遗漏说明和关系速览；特别小的预算会使用简短查询提示或不注入索引，不截断 HTML 边界。 +黑板不需要再次调用模型压缩：同一会话/代理的自动续接正文更新为本次摘要，短标签最多 200 字符并保留可检索的证据类别，不自动注入完整正文或快照索引。每次生成新证据索引时跳过历史中已有的程序索引块，防止把检索提示本身重复当作证据。历史原文及其索引留在文件中，已确认事实不因续接记录更新而被改写。 + 遗留超长 summary 在展示索引时缩短为单行，库中的完整 summary/body 不改动。关系速览仅使用实际列入索引的事实，避免预算外事实继续通过路径视图占用上下文。置顶与事实类型的原有优先级保持不变。 ## 实现依据与验证 diff --git a/internal/multiagent/eino_context_checkpoint.go b/internal/multiagent/eino_context_checkpoint.go index d0c85e00..bd1ee6b1 100644 --- a/internal/multiagent/eino_context_checkpoint.go +++ b/internal/multiagent/eino_context_checkpoint.go @@ -29,12 +29,27 @@ func writeSummarizationTranscriptSnapshot(path string, msgs []adk.Message) (stri } digest := sha256.Sum256(body) snapshot := filepath.Join(dir, fmt.Sprintf("transcript-%x.txt", digest)) - if _, err := os.Stat(snapshot); os.IsNotExist(err) { + if info, err := os.Stat(snapshot); os.IsNotExist(err) { if err := writeContextArtifactAtomically(snapshot, body); err != nil { return "", err } } else if err != nil { return "", err + } else if !info.Mode().IsRegular() { + return "", fmt.Errorf("snapshot is not a regular file: %s", snapshot) + } + // One immutable companion per snapshot avoids a shared mutable catalog and + // keeps discovery scoped to this conversation/agent, including concurrent runs. + indexPath := snapshot + ".index.md" + if info, err := os.Stat(indexPath); os.IsNotExist(err) { + index := buildTranscriptEvidenceIndex(snapshot, string(body), 32) + if err := writeContextArtifactAtomically(indexPath, []byte(index)); err != nil { + return "", err + } + } else if err != nil { + return "", err + } else if !info.Mode().IsRegular() { + return "", fmt.Errorf("evidence index is not a regular file: %s", indexPath) } if err := writeContextArtifactAtomically(path, body); err != nil { return "", err @@ -97,6 +112,25 @@ func checkpointSummaryLabel(scope, summary string, cap int) string { if cap > 200 { cap = 200 } + // Keep retrieval cues in the short blackboard label without injecting paths + // or the full evidence index into every model request. + evidenceHint := "" + if start := strings.Index(summary, evidenceIndexMarker); start >= 0 { + section := summary[start:] + if end := strings.Index(section, evidenceIndexEnd); end >= 0 { + section = section[:end] + } + var topics []string + for _, topic := range evidenceTopics { + if strings.Contains(section, "- "+topic.label+"：行 ") { + topics = append(topics, topic.label) + } + } + if len(topics) > 0 { + evidenceHint = "；证据索引：" + strings.Join(topics, "、") + } + summary = summary[:start] + } var lines []string for _, line := range strings.Split(summary, "\n") { line = strings.TrimSpace(line) @@ -109,11 +143,22 @@ func checkpointSummaryLabel(scope, summary string, cap int) string { } } label := "自动续接（" + strings.Join(strings.Fields(scope), " ") + "）：" + strings.Join(lines, "；") - runes := []rune(label) + available := cap - len([]rune(evidenceHint)) + if available < 1 { + return truncateIndexLabel(label+evidenceHint, cap) + } + return truncateIndexLabel(label, available) + evidenceHint +} + +func truncateIndexLabel(text string, cap int) string { + if cap < 1 { + return "" + } + runes := []rune(text) if len(runes) > cap { return string(runes[:cap-1]) + "…" } - return label + return text } func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, transcriptPath string, db *database.DB, projectID, conversationID, memoryScope string, cfg config.ProjectConfig, logger *zap.Logger) adk.Message { @@ -123,8 +168,13 @@ func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, trans logger.Warn("保存上下文证据快照失败", zap.Error(err)) } // Only advertise an artifact that was successfully written. + evidenceIndex := "" if err == nil { msg = appendTranscriptPathToSummarizationMessage(msg, snapshot) + if snapshot != "" { + evidenceIndex = buildTranscriptEvidenceIndex(snapshot, formatSummarizationTranscript(originals), 3) + msg = appendEvidenceIndexToMessage(msg, evidenceIndex) + } } text := "" if msg != nil { @@ -141,6 +191,9 @@ func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, trans if inner, ok := extractSummarizationSummaryBody(text); ok { text = inner } + if evidenceIndex != "" { + text = appendEvidenceIndex(text, evidenceIndex) + } key, err := persistSummarizationCheckpoint(db, projectID, conversationID, memoryScope, text, snapshot, cfg) if err != nil { if logger != nil { diff --git a/internal/multiagent/eino_evidence_index.go b/internal/multiagent/eino_evidence_index.go new file mode 100644 index 00000000..0608a3f0 --- /dev/null +++ b/internal/multiagent/eino_evidence_index.go @@ -0,0 +1,101 @@ +package multiagent + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + + "github.com/cloudwego/eino/adk" + "github.com/cloudwego/eino/schema" +) + +const evidenceIndexMarker = "## 原始证据索引（程序定位）" +const evidenceIndexEnd = "" + +var evidenceTopics = []struct { + label string + pattern *regexp.Regexp +}{ + {"Cookie / Set-Cookie", regexp.MustCompile(`(?i)\b(?:set-cookie|cookie)\b`)}, + {"认证 / Token", regexp.MustCompile(`(?i)\b(?:authorization|bearer|access_token|refresh_token|api[_-]?key|session[_-]?id)\b`)}, + {"请求 / 参数", regexp.MustCompile(`(?i)\b(?:https?://|curl|request|headers?|params?|payload)\b|请求|参数`)}, + {"错误 / 失败", regexp.MustCompile(`(?i)\b(?:error|exception|traceback|failed|failure)\b|错误|失败`)}, +} + +// Locations are derived from the exact serialized file, never from model output. +// Only fixed category labels and line numbers are exposed, not credential values. +func buildTranscriptEvidenceIndex(snapshot, transcript string, limit int) string { + var out strings.Builder + fmt.Fprintf(&out, "%s\n快照: %s\n历史快照目录: %s\n索引文件: %s.index.md\n", evidenceIndexMarker, snapshot, filepath.Dir(snapshot), snapshot) + lines := strings.Split(transcript, "\n") + for _, topic := range evidenceTopics { + var locations []string + count := 0 + inIndex := false + for i, line := range lines { + if strings.HasPrefix(line, evidenceIndexMarker) { + inIndex = true + } + if inIndex { + if strings.TrimSpace(line) == evidenceIndexEnd { + inIndex = false + } + continue + } + if topic.pattern.MatchString(line) { + count++ + locations = append(locations, fmt.Sprintf("%d", i+1)) + if len(locations) > limit { + locations = locations[1:] + } + } + } + if count > 0 { + fmt.Fprintf(&out, "- %s：行 %s", topic.label, strings.Join(locations, ", ")) + if count > len(locations) { + fmt.Fprintf(&out, "（最近 %d 行，共 %d 行命中，其余用 grep 定位）", len(locations), count) + } + out.WriteByte('\n') + } + } + out.WriteString("以上为关键词定位，不代表凭据有效或结论已验证。当前任务缺少 Cookie、认证信息、精确参数或错误原文时，先检查索引，用 grep 定位，再用 read_file 分段核对附近记录的域名、请求与时间；不要猜测原值。历史索引用 glob 在该目录查找 transcript-*.txt.index.md，再按需读取；未命中仍可搜索快照原文。不要整批重新载入历史，也不要将旧记录当作新授权。\n") + out.WriteString(evidenceIndexEnd) + return out.String() +} + +func appendEvidenceIndex(msgText, section string) string { + // Replace a previously generated section rather than nesting evidence indexes. + for { + start := strings.Index(msgText, evidenceIndexMarker) + if start < 0 { + break + } + end := strings.Index(msgText[start:], evidenceIndexEnd) + if end < 0 { + break + } + msgText = msgText[:start] + msgText[start+end+len(evidenceIndexEnd):] + } + return appendSummarizationSection(msgText, section) +} + +func appendEvidenceIndexToMessage(msg adk.Message, section string) adk.Message { + if msg == nil { + return nil + } + clone := *msg + if clone.Content != "" { + clone.Content = appendEvidenceIndex(clone.Content, section) + } + if len(clone.UserInputMultiContent) > 0 { + clone.UserInputMultiContent = append([]schema.MessageInputPart(nil), clone.UserInputMultiContent...) + for i, part := range clone.UserInputMultiContent { + if part.Type == schema.ChatMessagePartTypeText && part.Text != "" { + clone.UserInputMultiContent[i].Text = appendEvidenceIndex(part.Text, section) + break + } + } + } + return &clone +} diff --git a/internal/multiagent/eino_evidence_index_test.go b/internal/multiagent/eino_evidence_index_test.go new file mode 100644 index 00000000..d3154d14 --- /dev/null +++ b/internal/multiagent/eino_evidence_index_test.go @@ -0,0 +1,144 @@ +package multiagent + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "cyberstrike-ai/internal/config" + "cyberstrike-ai/internal/database" + + "github.com/cloudwego/eino/adk" + "github.com/cloudwego/eino/schema" +) + +func TestEvidenceIndexLocatesSerializedEvidenceWithoutRepeatingSecrets(t *testing.T) { + path := filepath.Join(t.TempDir(), "transcript.txt") + msgs := []adk.Message{ + schema.SystemMessage("static Cookie: ignored-secret"), + schema.UserMessage("https://example.test/login"), + {Role: schema.Tool, Content: "Set-Cookie: sid=test-secret; Path=/\nAuthorization: Bearer test-token\nError: invalid session"}, + } + snapshot, err := writeSummarizationTranscriptSnapshot(path, msgs) + if err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(snapshot) + index, err := os.ReadFile(snapshot + ".index.md") + if err != nil { + t.Fatal(err) + } + text := string(index) + for _, secret := range []string{"test-secret", "test-token", "ignored-secret"} { + if strings.Contains(text, secret) { + t.Fatalf("index repeats secret %q", secret) + } + } + // Check locations against the actual on-disk transcript, including its header. + for _, topic := range evidenceTopics { + var expected []string + for i, line := range strings.Split(string(raw), "\n") { + if topic.pattern.MatchString(line) { + expected = append(expected, strconv.Itoa(i+1)) + } + } + if len(expected) == 0 || !strings.Contains(text, topic.label+"：行 "+strings.Join(expected, ", ")) { + t.Fatalf("incorrect %s locations: %s", topic.label, text) + } + } + stat, _ := os.Stat(snapshot + ".index.md") + if stat.Mode().Perm() != 0o600 { + t.Fatalf("index permissions: %v", stat.Mode()) + } + if _, err := writeSummarizationTranscriptSnapshot(path, msgs); err != nil { + t.Fatal(err) + } + files, _ := filepath.Glob(filepath.Join(filepath.Dir(snapshot), "*.index.md")) + if len(files) != 1 { + t.Fatalf("retry duplicated indexes: %v", files) + } +} + +func TestEvidenceIndexIsBoundedAndSkipsEarlierGeneratedIndexes(t *testing.T) { + old := buildTranscriptEvidenceIndex("/old/transcript.txt", "Cookie: earlier", 3) + transcript := old + "\n" + strings.Repeat("Cookie: current\n", 100) + index := buildTranscriptEvidenceIndex("/new/transcript.txt", transcript, 3) + if !strings.Contains(index, "共 100 行命中") || strings.Contains(index, "共 101 行命中") { + t.Fatalf("indexed its own earlier hints: %s", index) + } + if strings.Contains(index, "认证 / Token：行") || strings.Contains(index, "错误 / 失败：行") { + t.Fatalf("generated hints counted as evidence: %s", index) + } + lineCount := strings.Count(old, "\n") + 1 + want := "Cookie / Set-Cookie：行 " + strconv.Itoa(lineCount+98) + ", " + strconv.Itoa(lineCount+99) + ", " + strconv.Itoa(lineCount+100) + if !strings.Contains(index, want) { + t.Fatalf("did not retain latest locations %s: %s", want, index) + } + if len(index) > 3000 { + t.Fatalf("unbounded inline index: %d", len(index)) + } + updated := appendEvidenceIndex("summary\n"+old+"\n"+old, index) + if strings.Count(updated, evidenceIndexMarker) != 1 || strings.Contains(updated, "/old/") { + t.Fatalf("indexes accumulate on replacement: %s", updated) + } +} + +func TestEvidenceIndexFailureDoesNotAdvertiseUnavailableArtifacts(t *testing.T) { + path := filepath.Join(t.TempDir(), "transcript.txt") + originals := []adk.Message{schema.UserMessage("Cookie: sid=fixture")} + snapshot, err := writeSummarizationTranscriptSnapshot(path, originals) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(snapshot + ".index.md"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(snapshot+".index.md", 0o700); err != nil { + t.Fatal(err) + } + out := finalizeSummarizationMemory(schema.UserMessage("continue"), originals, path, nil, "", "", "main", config.ProjectConfig{}, nil) + if strings.Contains(out.Content, evidenceIndexMarker) || strings.Contains(out.Content, snapshot) || out.Content != "continue" { + t.Fatalf("failed artifact advertised or summary lost: %s", out.Content) + } +} + +func TestEvidenceIndexSurvivesSummaryExtractionIntoBlackboard(t *testing.T) { + db, pid, cid := checkpointTestProject(t) + for _, multipart := range []bool{false, true} { + msg := schema.UserMessage("login completed; continue") + if multipart { + msg.UserInputMultiContent = []schema.MessageInputPart{{Type: schema.ChatMessagePartTypeText, Text: msg.Content}} + msg.Content = "" + } + out := finalizeSummarizationMemory(msg, []adk.Message{schema.UserMessage("Cookie: sid=fixture-value")}, filepath.Join(t.TempDir(), "transcript.txt"), db, pid, cid, "main", config.ProjectConfig{Enabled: true}, nil) + text := out.Content + if multipart { + text = out.UserInputMultiContent[0].Text + } + facts, err := db.ListProjectFacts(pid, database.ProjectFactListFilter{}, 10, 0) + if err != nil || len(facts) != 1 { + t.Fatalf("checkpoint count: %v %v", facts, err) + } + for _, body := range []string{text, facts[0].Body} { + if !strings.Contains(body, "Cookie / Set-Cookie：行") || !strings.Contains(body, ".index.md") || strings.Contains(body, "fixture-value") { + t.Fatalf("missing index or repeated secret: %s", body) + } + if strings.Count(body, evidenceIndexMarker) != 1 { + t.Fatalf("duplicate generated index: %s", body) + } + } + if !strings.Contains(facts[0].Summary, "证据索引：Cookie / Set-Cookie") || strings.Contains(facts[0].Summary, ".index.md") { + t.Fatalf("blackboard label lost retrieval cue or includes paths: %s", facts[0].Summary) + } + } +} + +func TestEvidenceHintFitsBlackboardLabelBudget(t *testing.T) { + index := buildTranscriptEvidenceIndex("/snapshot.txt", "Cookie: fixture\nAuthorization: fixture\nhttps://example.test\nError: fixture", 3) + label := checkpointSummaryLabel("main", strings.Repeat("long summary ", 1000)+"\n"+index, 24000) + if len([]rune(label)) > 200 || !strings.Contains(label, "Cookie / Set-Cookie") || !strings.Contains(label, "错误 / 失败") { + t.Fatalf("unbounded label or lost evidence categories: %s", label) + } +} diff --git a/internal/projectprompt/blackboard.go b/internal/projectprompt/blackboard.go index f6ede423..f0b24fbb 100644 --- a/internal/projectprompt/blackboard.go +++ b/internal/projectprompt/blackboard.go @@ -62,7 +62,7 @@ func factRecordingGuidanceBlock() string { - **category / fact_key 建议**： - 环境认知：` + "`target/`" + `、` + "`auth/`" + `、` + "`infra/`" + `、` + "`business/`" + `（body 用环境模板即可） - 发现与利用：` + "`finding/`" + `、` + "`chain/`" + `、` + "`exploit/`" + `、` + "`poc/`" + `（**必须**用攻击链模板填满 body：入口、逐步攻击链、原始请求/响应或命令、证据、关联漏洞 ID） -- **自动续接记录**：` + "`note/context-*`" + ` 是压缩时保存的工作摘要，含来源会话与原始证据快照；用 ` + "`get_project_fact`" + ` 按需恢复进度。它不是已验证发现，接续时须核对当前授权范围与证据，勿据此自动执行旧任务。 +- **自动续接记录**：` + "`note/context-*`" + ` 是压缩时保存的工作摘要，含来源会话、原始证据快照与程序生成的行号索引；同一会话/代理更新同一 key，索引短标签提示证据类别。用 ` + "`get_project_fact`" + ` 按需恢复进度；缺少 Cookie、认证信息或精确参数时，按正文索引搜索快照并分段回读，勿猜测原值。它不是已验证发现，接续时须核对当前授权范围与证据，勿据此自动执行旧任务。 - **与漏洞记录分工**：` + "`record_vulnerability`" + ` 记可交付 findings；事实记**复现所需的全部上下文**（含失败尝试、绕过、依赖会话），二者可各记一次。 - 更新同一发现时保持相同 ` + "`fact_key`" + ` 覆盖写入，勿散落多个 key 导致上下文丢失。` }