feat: index compaction evidence for targeted context recovery

docs/zh-CN/blackboard-context-memory.md

@@ -10,12 +10,28 @@
 
 压缩后的上下文会附上该记录的 `get_project_fact(key)` 入口，并刷新系统中的黑板索引。后续按需读取记录，再使用 `read_file` 分段查看原始证据；无须重新把完整历史注入每一次模型请求。
 
+新上下文和黑板记录都包含程序生成的证据索引：按实际写入文件的行号标出 Cookie / Set-Cookie、认证 / Token、请求 / 参数、错误 / 失败的关键词命中位置。索引只包含固定类别与位置，不复制凭据值；它是检索线索，不代表相关凭据仍有效或结论已验证。需要精确值而摘要缺失时，Agent 应先检查索引，使用 `grep` 定位，再用 `read_file` 分段核对附近记录的域名、请求和时间，不猜测原值。
+
+每类在上下文中最多列出最近 3 个命中行号，完整索引文件每类最多列出最近 32 个；超出时提示总命中数与搜索入口。关键词索引不是完整语义索引，未命中也可以搜索快照原文。
+
 ## 原始证据不会随下一次压缩被覆盖
 
 过去所有压缩轮次写入同一个 `transcript.txt`，后一次写入会覆盖前一次记录。现在每份压缩前历史按内容 SHA-256 保存到 `snapshots/transcript-<hash>.txt`，相同内容的重试复用快照，不重复建文件。`transcript.txt` 仍保留为最新记录的兼容入口，续接摘要引用的是具体快照。快照沿用现有会话记录格式，保留用户、助手与工具记录，省略不变的系统提示与工具/技能索引。
 
 快照沿用 `conversation_artifacts/<conversationId>/summarization/` 目录，与会话产物一起管理；子代理使用独立 scope 子目录。删除会话时现有清理逻辑会删除对应产物，快照不作为会话删除后的独立备份。文件通过临时文件原子替换写入，权限为 0600。
 
+默认数据库为 `data/conversations.db` 时，主代理文件布局如下（摘要向 Agent 提供绝对路径）：
+
+```text
+data/conversation_artifacts/<conversationId>/summarization/
+├── transcript.txt
+└── snapshots/
+    ├── transcript-<hash>.txt
+    └── transcript-<hash>.txt.index.md
+```
+
+子代理、规划器和执行器使用 `summarization/scopes/<scopeHash>/snapshots/`。每份快照有独立索引文件，相同内容重试复用同一文件；历史索引不随最新记录更新而消失。摘要提供历史快照目录，Agent 可用 `glob` 查找 `transcript-*.txt.index.md`，按需查找旧证据，无须维护多个代理并发覆盖的共享目录清单。
+
 系统复用本次已经生成的压缩摘要，不增加一次额外的模型提取调用。原始历史在生成摘要前就做快照，因此即使摘要遗漏某个精确细节，也还有来源可以追溯。黑板或快照保存失败时会记录告警，保留当前可用摘要继续运行，不提示保存成功。
 
 ## 配置和边界
@@ -36,6 +52,8 @@ project:
 
 自动注入仍只包含 key、短摘要与关系边，完整 body 留在数据库按需读取。索引查询不再加载最终不会注入的 body。字符预算现在包含标题、起止标记、查询提示、遗漏说明和关系速览；特别小的预算会使用简短查询提示或不注入索引，不截断 HTML 边界。
 
+黑板不需要再次调用模型压缩：同一会话/代理的自动续接正文更新为本次摘要，短标签最多 200 字符并保留可检索的证据类别，不自动注入完整正文或快照索引。每次生成新证据索引时跳过历史中已有的程序索引块，防止把检索提示本身重复当作证据。历史原文及其索引留在文件中，已确认事实不因续接记录更新而被改写。
+
 遗留超长 summary 在展示索引时缩短为单行，库中的完整 summary/body 不改动。关系速览仅使用实际列入索引的事实，避免预算外事实继续通过路径视图占用上下文。置顶与事实类型的原有优先级保持不变。
 
 ## 实现依据与验证

internal/multiagent/eino_context_checkpoint.go

@@ -29,12 +29,27 @@ func writeSummarizationTranscriptSnapshot(path string, msgs []adk.Message) (stri
 	}
 	digest := sha256.Sum256(body)
 	snapshot := filepath.Join(dir, fmt.Sprintf("transcript-%x.txt", digest))
-	if _, err := os.Stat(snapshot); os.IsNotExist(err) {
+	if info, err := os.Stat(snapshot); os.IsNotExist(err) {
 		if err := writeContextArtifactAtomically(snapshot, body); err != nil {
 			return "", err
 		}
 	} else if err != nil {
 		return "", err
+	} else if !info.Mode().IsRegular() {
+		return "", fmt.Errorf("snapshot is not a regular file: %s", snapshot)
+	}
+	// One immutable companion per snapshot avoids a shared mutable catalog and
+	// keeps discovery scoped to this conversation/agent, including concurrent runs.
+	indexPath := snapshot + ".index.md"
+	if info, err := os.Stat(indexPath); os.IsNotExist(err) {
+		index := buildTranscriptEvidenceIndex(snapshot, string(body), 32)
+		if err := writeContextArtifactAtomically(indexPath, []byte(index)); err != nil {
+			return "", err
+		}
+	} else if err != nil {
+		return "", err
+	} else if !info.Mode().IsRegular() {
+		return "", fmt.Errorf("evidence index is not a regular file: %s", indexPath)
 	}
 	if err := writeContextArtifactAtomically(path, body); err != nil {
 		return "", err
@@ -97,6 +112,25 @@ func checkpointSummaryLabel(scope, summary string, cap int) string {
 	if cap > 200 {
 		cap = 200
 	}
+	// Keep retrieval cues in the short blackboard label without injecting paths
+	// or the full evidence index into every model request.
+	evidenceHint := ""
+	if start := strings.Index(summary, evidenceIndexMarker); start >= 0 {
+		section := summary[start:]
+		if end := strings.Index(section, evidenceIndexEnd); end >= 0 {
+			section = section[:end]
+		}
+		var topics []string
+		for _, topic := range evidenceTopics {
+			if strings.Contains(section, "- "+topic.label+"：行 ") {
+				topics = append(topics, topic.label)
+			}
+		}
+		if len(topics) > 0 {
+			evidenceHint = "；证据索引：" + strings.Join(topics, "、")
+		}
+		summary = summary[:start]
+	}
 	var lines []string
 	for _, line := range strings.Split(summary, "\n") {
 		line = strings.TrimSpace(line)
@@ -109,11 +143,22 @@ func checkpointSummaryLabel(scope, summary string, cap int) string {
 		}
 	}
 	label := "自动续接（" + strings.Join(strings.Fields(scope), " ") + "）：" + strings.Join(lines, "；")
-	runes := []rune(label)
+	available := cap - len([]rune(evidenceHint))
+	if available < 1 {
+		return truncateIndexLabel(label+evidenceHint, cap)
+	}
+	return truncateIndexLabel(label, available) + evidenceHint
+}
+
+func truncateIndexLabel(text string, cap int) string {
+	if cap < 1 {
+		return ""
+	}
+	runes := []rune(text)
 	if len(runes) > cap {
 		return string(runes[:cap-1]) + "…"
 	}
-	return label
+	return text
 }
 
 func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, transcriptPath string, db *database.DB, projectID, conversationID, memoryScope string, cfg config.ProjectConfig, logger *zap.Logger) adk.Message {
@@ -123,8 +168,13 @@ func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, trans
 		logger.Warn("保存上下文证据快照失败", zap.Error(err))
 	}
 	// Only advertise an artifact that was successfully written.
+	evidenceIndex := ""
 	if err == nil {
 		msg = appendTranscriptPathToSummarizationMessage(msg, snapshot)
+		if snapshot != "" {
+			evidenceIndex = buildTranscriptEvidenceIndex(snapshot, formatSummarizationTranscript(originals), 3)
+			msg = appendEvidenceIndexToMessage(msg, evidenceIndex)
+		}
 	}
 	text := ""
 	if msg != nil {
@@ -141,6 +191,9 @@ func finalizeSummarizationMemory(msg adk.Message, originals []adk.Message, trans
 	if inner, ok := extractSummarizationSummaryBody(text); ok {
 		text = inner
 	}
+	if evidenceIndex != "" {
+		text = appendEvidenceIndex(text, evidenceIndex)
+	}
 	key, err := persistSummarizationCheckpoint(db, projectID, conversationID, memoryScope, text, snapshot, cfg)
 	if err != nil {
 		if logger != nil {

internal/multiagent/eino_evidence_index.go

@@ -0,0 +1,101 @@
+package multiagent
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/cloudwego/eino/adk"
+	"github.com/cloudwego/eino/schema"
+)
+
+const evidenceIndexMarker = "## 原始证据索引（程序定位）"
+const evidenceIndexEnd = "<!-- evidence-index-end -->"
+
+var evidenceTopics = []struct {
+	label   string
+	pattern *regexp.Regexp
+}{
+	{"Cookie / Set-Cookie", regexp.MustCompile(`(?i)\b(?:set-cookie|cookie)\b`)},
+	{"认证 / Token", regexp.MustCompile(`(?i)\b(?:authorization|bearer|access_token|refresh_token|api[_-]?key|session[_-]?id)\b`)},
+	{"请求 / 参数", regexp.MustCompile(`(?i)\b(?:https?://|curl|request|headers?|params?|payload)\b|请求|参数`)},
+	{"错误 / 失败", regexp.MustCompile(`(?i)\b(?:error|exception|traceback|failed|failure)\b|错误|失败`)},
+}
+
+// Locations are derived from the exact serialized file, never from model output.
+// Only fixed category labels and line numbers are exposed, not credential values.
+func buildTranscriptEvidenceIndex(snapshot, transcript string, limit int) string {
+	var out strings.Builder
+	fmt.Fprintf(&out, "%s\n快照: %s\n历史快照目录: %s\n索引文件: %s.index.md\n", evidenceIndexMarker, snapshot, filepath.Dir(snapshot), snapshot)
+	lines := strings.Split(transcript, "\n")
+	for _, topic := range evidenceTopics {
+		var locations []string
+		count := 0
+		inIndex := false
+		for i, line := range lines {
+			if strings.HasPrefix(line, evidenceIndexMarker) {
+				inIndex = true
+			}
+			if inIndex {
+				if strings.TrimSpace(line) == evidenceIndexEnd {
+					inIndex = false
+				}
+				continue
+			}
+			if topic.pattern.MatchString(line) {
+				count++
+				locations = append(locations, fmt.Sprintf("%d", i+1))
+				if len(locations) > limit {
+					locations = locations[1:]
+				}
+			}
+		}
+		if count > 0 {
+			fmt.Fprintf(&out, "- %s：行 %s", topic.label, strings.Join(locations, ", "))
+			if count > len(locations) {
+				fmt.Fprintf(&out, "（最近 %d 行，共 %d 行命中，其余用 grep 定位）", len(locations), count)
+			}
+			out.WriteByte('\n')
+		}
+	}
+	out.WriteString("以上为关键词定位，不代表凭据有效或结论已验证。当前任务缺少 Cookie、认证信息、精确参数或错误原文时，先检查索引，用 grep 定位，再用 read_file 分段核对附近记录的域名、请求与时间；不要猜测原值。历史索引用 glob 在该目录查找 transcript-*.txt.index.md，再按需读取；未命中仍可搜索快照原文。不要整批重新载入历史，也不要将旧记录当作新授权。\n")
+	out.WriteString(evidenceIndexEnd)
+	return out.String()
+}
+
+func appendEvidenceIndex(msgText, section string) string {
+	// Replace a previously generated section rather than nesting evidence indexes.
+	for {
+		start := strings.Index(msgText, evidenceIndexMarker)
+		if start < 0 {
+			break
+		}
+		end := strings.Index(msgText[start:], evidenceIndexEnd)
+		if end < 0 {
+			break
+		}
+		msgText = msgText[:start] + msgText[start+end+len(evidenceIndexEnd):]
+	}
+	return appendSummarizationSection(msgText, section)
+}
+
+func appendEvidenceIndexToMessage(msg adk.Message, section string) adk.Message {
+	if msg == nil {
+		return nil
+	}
+	clone := *msg
+	if clone.Content != "" {
+		clone.Content = appendEvidenceIndex(clone.Content, section)
+	}
+	if len(clone.UserInputMultiContent) > 0 {
+		clone.UserInputMultiContent = append([]schema.MessageInputPart(nil), clone.UserInputMultiContent...)
+		for i, part := range clone.UserInputMultiContent {
+			if part.Type == schema.ChatMessagePartTypeText && part.Text != "" {
+				clone.UserInputMultiContent[i].Text = appendEvidenceIndex(part.Text, section)
+				break
+			}
+		}
+	}
+	return &clone
+}

internal/multiagent/eino_evidence_index_test.go

@@ -0,0 +1,144 @@
+package multiagent
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"cyberstrike-ai/internal/config"
+	"cyberstrike-ai/internal/database"
+
+	"github.com/cloudwego/eino/adk"
+	"github.com/cloudwego/eino/schema"
+)
+
+func TestEvidenceIndexLocatesSerializedEvidenceWithoutRepeatingSecrets(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "transcript.txt")
+	msgs := []adk.Message{
+		schema.SystemMessage("static Cookie: ignored-secret"),
+		schema.UserMessage("https://example.test/login"),
+		{Role: schema.Tool, Content: "Set-Cookie: sid=test-secret; Path=/\nAuthorization: Bearer test-token\nError: invalid session"},
+	}
+	snapshot, err := writeSummarizationTranscriptSnapshot(path, msgs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, _ := os.ReadFile(snapshot)
+	index, err := os.ReadFile(snapshot + ".index.md")
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := string(index)
+	for _, secret := range []string{"test-secret", "test-token", "ignored-secret"} {
+		if strings.Contains(text, secret) {
+			t.Fatalf("index repeats secret %q", secret)
+		}
+	}
+	// Check locations against the actual on-disk transcript, including its header.
+	for _, topic := range evidenceTopics {
+		var expected []string
+		for i, line := range strings.Split(string(raw), "\n") {
+			if topic.pattern.MatchString(line) {
+				expected = append(expected, strconv.Itoa(i+1))
+			}
+		}
+		if len(expected) == 0 || !strings.Contains(text, topic.label+"：行 "+strings.Join(expected, ", ")) {
+			t.Fatalf("incorrect %s locations: %s", topic.label, text)
+		}
+	}
+	stat, _ := os.Stat(snapshot + ".index.md")
+	if stat.Mode().Perm() != 0o600 {
+		t.Fatalf("index permissions: %v", stat.Mode())
+	}
+	if _, err := writeSummarizationTranscriptSnapshot(path, msgs); err != nil {
+		t.Fatal(err)
+	}
+	files, _ := filepath.Glob(filepath.Join(filepath.Dir(snapshot), "*.index.md"))
+	if len(files) != 1 {
+		t.Fatalf("retry duplicated indexes: %v", files)
+	}
+}
+
+func TestEvidenceIndexIsBoundedAndSkipsEarlierGeneratedIndexes(t *testing.T) {
+	old := buildTranscriptEvidenceIndex("/old/transcript.txt", "Cookie: earlier", 3)
+	transcript := old + "\n" + strings.Repeat("Cookie: current\n", 100)
+	index := buildTranscriptEvidenceIndex("/new/transcript.txt", transcript, 3)
+	if !strings.Contains(index, "共 100 行命中") || strings.Contains(index, "共 101 行命中") {
+		t.Fatalf("indexed its own earlier hints: %s", index)
+	}
+	if strings.Contains(index, "认证 / Token：行") || strings.Contains(index, "错误 / 失败：行") {
+		t.Fatalf("generated hints counted as evidence: %s", index)
+	}
+	lineCount := strings.Count(old, "\n") + 1
+	want := "Cookie / Set-Cookie：行 " + strconv.Itoa(lineCount+98) + ", " + strconv.Itoa(lineCount+99) + ", " + strconv.Itoa(lineCount+100)
+	if !strings.Contains(index, want) {
+		t.Fatalf("did not retain latest locations %s: %s", want, index)
+	}
+	if len(index) > 3000 {
+		t.Fatalf("unbounded inline index: %d", len(index))
+	}
+	updated := appendEvidenceIndex("summary\n"+old+"\n"+old, index)
+	if strings.Count(updated, evidenceIndexMarker) != 1 || strings.Contains(updated, "/old/") {
+		t.Fatalf("indexes accumulate on replacement: %s", updated)
+	}
+}
+
+func TestEvidenceIndexFailureDoesNotAdvertiseUnavailableArtifacts(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "transcript.txt")
+	originals := []adk.Message{schema.UserMessage("Cookie: sid=fixture")}
+	snapshot, err := writeSummarizationTranscriptSnapshot(path, originals)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(snapshot + ".index.md"); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(snapshot+".index.md", 0o700); err != nil {
+		t.Fatal(err)
+	}
+	out := finalizeSummarizationMemory(schema.UserMessage("continue"), originals, path, nil, "", "", "main", config.ProjectConfig{}, nil)
+	if strings.Contains(out.Content, evidenceIndexMarker) || strings.Contains(out.Content, snapshot) || out.Content != "continue" {
+		t.Fatalf("failed artifact advertised or summary lost: %s", out.Content)
+	}
+}
+
+func TestEvidenceIndexSurvivesSummaryExtractionIntoBlackboard(t *testing.T) {
+	db, pid, cid := checkpointTestProject(t)
+	for _, multipart := range []bool{false, true} {
+		msg := schema.UserMessage("<summary>login completed; continue</summary>")
+		if multipart {
+			msg.UserInputMultiContent = []schema.MessageInputPart{{Type: schema.ChatMessagePartTypeText, Text: msg.Content}}
+			msg.Content = ""
+		}
+		out := finalizeSummarizationMemory(msg, []adk.Message{schema.UserMessage("Cookie: sid=fixture-value")}, filepath.Join(t.TempDir(), "transcript.txt"), db, pid, cid, "main", config.ProjectConfig{Enabled: true}, nil)
+		text := out.Content
+		if multipart {
+			text = out.UserInputMultiContent[0].Text
+		}
+		facts, err := db.ListProjectFacts(pid, database.ProjectFactListFilter{}, 10, 0)
+		if err != nil || len(facts) != 1 {
+			t.Fatalf("checkpoint count: %v %v", facts, err)
+		}
+		for _, body := range []string{text, facts[0].Body} {
+			if !strings.Contains(body, "Cookie / Set-Cookie：行") || !strings.Contains(body, ".index.md") || strings.Contains(body, "fixture-value") {
+				t.Fatalf("missing index or repeated secret: %s", body)
+			}
+			if strings.Count(body, evidenceIndexMarker) != 1 {
+				t.Fatalf("duplicate generated index: %s", body)
+			}
+		}
+		if !strings.Contains(facts[0].Summary, "证据索引：Cookie / Set-Cookie") || strings.Contains(facts[0].Summary, ".index.md") {
+			t.Fatalf("blackboard label lost retrieval cue or includes paths: %s", facts[0].Summary)
+		}
+	}
+}
+
+func TestEvidenceHintFitsBlackboardLabelBudget(t *testing.T) {
+	index := buildTranscriptEvidenceIndex("/snapshot.txt", "Cookie: fixture\nAuthorization: fixture\nhttps://example.test\nError: fixture", 3)
+	label := checkpointSummaryLabel("main", strings.Repeat("long summary ", 1000)+"\n"+index, 24000)
+	if len([]rune(label)) > 200 || !strings.Contains(label, "Cookie / Set-Cookie") || !strings.Contains(label, "错误 / 失败") {
+		t.Fatalf("unbounded label or lost evidence categories: %s", label)
+	}
+}

internal/projectprompt/blackboard.go

@@ -62,7 +62,7 @@ func factRecordingGuidanceBlock() string {
 - **category / fact_key 建议**：
   - 环境认知：` + "`target/`" + `、` + "`auth/`" + `、` + "`infra/`" + `、` + "`business/`" + `（body 用环境模板即可）
   - 发现与利用：` + "`finding/`" + `、` + "`chain/`" + `、` + "`exploit/`" + `、` + "`poc/`" + `（**必须**用攻击链模板填满 body：入口、逐步攻击链、原始请求/响应或命令、证据、关联漏洞 ID）
-- **自动续接记录**：` + "`note/context-*`" + ` 是压缩时保存的工作摘要，含来源会话与原始证据快照；用 ` + "`get_project_fact`" + ` 按需恢复进度。它不是已验证发现，接续时须核对当前授权范围与证据，勿据此自动执行旧任务。
+- **自动续接记录**：` + "`note/context-*`" + ` 是压缩时保存的工作摘要，含来源会话、原始证据快照与程序生成的行号索引；同一会话/代理更新同一 key，索引短标签提示证据类别。用 ` + "`get_project_fact`" + ` 按需恢复进度；缺少 Cookie、认证信息或精确参数时，按正文索引搜索快照并分段回读，勿猜测原值。它不是已验证发现，接续时须核对当前授权范围与证据，勿据此自动执行旧任务。
 - **与漏洞记录分工**：` + "`record_vulnerability`" + ` 记可交付 findings；事实记**复现所需的全部上下文**（含失败尝试、绕过、依赖会话），二者可各记一次。
 - 更新同一发现时保持相同 ` + "`fact_key`" + ` 覆盖写入，勿散落多个 key 导致上下文丢失。`
 }