From f5e603c75e5bd0e6a31a0bbee8d3e7e670eda8da Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 11:00:46 +0000 Subject: [PATCH] fix(autoplan): CAP titles on different paths never match by title in reconcile or confirm rounds (wave review) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/autoplan-confirm.ts | 4 ++-- lib/autoplan-reconcile.ts | 13 +++++++++---- test/autoplan-pairing.test.ts | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/autoplan-confirm.ts b/lib/autoplan-confirm.ts index 427a9d502..b99ec628e 100644 --- a/lib/autoplan-confirm.ts +++ b/lib/autoplan-confirm.ts @@ -13,7 +13,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import type { FindingRow } from './headless-artifacts'; import { readJsonl } from './headless-artifacts'; -import { jaccard, tokens } from './autoplan-reconcile'; +import { titleScore } from './autoplan-reconcile'; import { readDecisions, type PhaseState, type RunState } from './autoplan-state'; export interface DiffLine { op: ' ' | '+' | '-'; text: string; a?: number; b?: number } @@ -135,7 +135,7 @@ export function annotateConfirm(state: RunState, phase: string, rows: FindingRow const isPlan = (f: FindingRow) => f.file === ctx.planName; let correlated = 0, settledCount = 0; for (const row of rows as Array) { - const same = earlier.filter(e => jaccard(tokens(e.title), tokens(row.title)) >= 0.6 || (!!e.file && !isPlan(e) && e.file === row.file && e.line === row.line)); + const same = earlier.filter(e => titleScore(e.title, row.title) >= 0.6 || (!!e.file && !isPlan(e) && e.file === row.file && e.line === row.line)); if (!same.length) continue; row.correlates = same.map(e => e.id); correlated++; diff --git a/lib/autoplan-reconcile.ts b/lib/autoplan-reconcile.ts index 76cfa03aa..fe6c9f363 100644 --- a/lib/autoplan-reconcile.ts +++ b/lib/autoplan-reconcile.ts @@ -211,6 +211,13 @@ export function jaccard(a: Set, b: Set): number { for (const t of a) if (b.has(t)) inter++; return inter / (a.size + b.size - inter); } +const capPath = (title: string) => /^\s*CAP[46]\s*:([^:]*):/i.exec(title)?.[1]!.trim(); +/** Title similarity. Two CAP-coded titles on different paths name different problems, so they never match. */ +export function titleScore(a: string, b: string): number { + const pa = capPath(a), pb = capPath(b); + if (pa !== undefined && pb !== undefined && pa !== pb) return 0; + return jaccard(tokens(a), tokens(b)); +} const level = (s: Severity) => SEVERITIES.indexOf(s); /** A remedy compared as words: case, punctuation and spacing ignored. */ export const remedyKey = (text: string | undefined) => (text ?? '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); @@ -265,12 +272,11 @@ export function reconcile(native: FindingRow[], outside: FindingRow[], overrides continue; } const near = (n: FindingRow) => !!o.file && o.line !== undefined && n.file === o.file && n.line !== undefined && Math.abs(n.line - o.line) <= 3; - const ot = tokens(o.title); let match: FindingRow | undefined; let best = 0; for (const n of native) { if (used.has(n.id) || (o.file && n.file && o.file !== n.file)) continue; - const score = jaccard(ot, tokens(n.title)) + (near(n) ? 0.01 : 0); + const score = titleScore(o.title, n.title) + (near(n) ? 0.01 : 0); if (score >= 0.5 && score > best) { best = score; match = n; } } const titled = !!match; @@ -294,9 +300,8 @@ export function reconcile(native: FindingRow[], outside: FindingRow[], overrides } // Plan reviews name a file without a line: same file, a line missing on either side, once every title and line match is placed. for (const [i, o] of later) { - const ot = tokens(o.title); const match = native.filter(n => !used.has(n.id) && n.file === o.file && (o.line === undefined || n.line === undefined)) - .reduce((b, n) => (!b || jaccard(ot, tokens(n.title)) > jaccard(ot, tokens(b.title)) ? n : b), undefined); + .reduce((b, n) => (!b || titleScore(o.title, n.title) > titleScore(o.title, b.title) ? n : b), undefined); if (!match) continue; used.add(match.id); rows[i] = { outside_id: o.id, native_id: match.id, disposition: 'unverified', resolution: `same file, agreement unverified (${o.file}): ${match.title}`, basis: 'file', severity: { outside: o.severity, native: match.severity } }; diff --git a/test/autoplan-pairing.test.ts b/test/autoplan-pairing.test.ts index 60c39a1ab..e630772ab 100644 --- a/test/autoplan-pairing.test.ts +++ b/test/autoplan-pairing.test.ts @@ -7,7 +7,7 @@ */ import { describe, expect, test } from 'bun:test'; import fixture from './fixtures/autoplan-pairing/run-20261014-pairs.json'; -import { reconcile } from '../lib/autoplan-reconcile'; +import { reconcile, titleScore } from '../lib/autoplan-reconcile'; import { challengeDefault } from '../lib/autoplan-gate'; const pair = (name: string) => fixture.pairs.find(p => p.name.startsWith(name))! as any; @@ -104,4 +104,17 @@ describe('plan reviews name a file without a line (GBRA-223 claim 1)', () => { const b = row('outside', 1, { severity: 'High', title: 'CAP6:lib/telemetry-sync.ts:recovery no retry bound on network failure', file: 'lib/telemetry-sync.ts' }); expect(reconcile([a], [b]).rows[0]).toMatchObject({ disposition: 'new', native_id: null }); }); + + test('two CAP titles on different paths never match by title, even when both findings sit on the plan file', () => { + const A = 'CAP6:bin/gstack-eval-plan pilot start:fix no fix line names the command'; + const B = 'CAP6:bin/gstack-outside-voice run:fix no fix line names the command'; + expect(titleScore(A, B)).toBe(0); + expect(titleScore(A, A)).toBe(1); + expect(titleScore(A, 'no fix line names the command')).toBe(1); + const n = row('native', 1, { severity: 'High', title: A, file: 'plan.md', resolution: 'name the command' }); + const o = row('outside', 1, { severity: 'High', title: B, file: 'plan.md', resolution: 'name the command' }); + expect(reconcile([n], [o]).rows[0]).toMatchObject({ basis: 'file', disposition: 'unverified' }); + const bare = (x: any) => { const { file, ...rest } = x; return rest; }; + expect(reconcile([bare(n)], [bare(o)]).rows[0]).toMatchObject({ disposition: 'new', native_id: null }); + }); });