diff --git a/changelog.d/gsta259-3925-unverified-verdict.md b/changelog.d/gsta259-3925-unverified-verdict.md new file mode 100644 index 000000000..9e70c40b5 --- /dev/null +++ b/changelog.d/gsta259-3925-unverified-verdict.md @@ -0,0 +1,4 @@ +### Fixed + +- **An outside review that says it could not verify the change is no longer read as a pass (#3925).** `lib/outside-review-result.ts` printed `VERDICT: clean` / `FINDINGS: none` for a Codex reply ending `Verdict: Can't verify (one changed file excluded and four required suites blocked)`. A last `Verdict:` line now has to say `ready` (or `not ready`, which still blocks) for the reply to count; anything else, and a `ready` that says something could not be verified, is `VERDICT: unverified` with the new `REASON: unverified_verdict` and exit 4. See docs/troubleshooting.md#outside-review-unverified-verdict. +- **A numbered finding without a P0-P3 tag is no longer dropped (#3925).** A reply with `1. **Nonblocking — x**`, `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)` printed `FINDINGS: none`. In a reply with a `Verdict:` line, every numbered item is now counted as a finding, and one with no readable severity makes `FINDINGS:` print `unspecified`. The verdict is unchanged (`clean` here, since the reviewer said ready). A consult answer with no `Verdict:` line, and a real clean reply, still read `clean` / `none`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f17f55959..9beeefa94 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -179,7 +179,14 @@ the readiness dashboard and the PR body, and never count it as a pass. A reply whose last `Verdict:` line says `not ready` is `findings` and exits 3 even when readable advisory severities (P2 or P3) were parsed; `FINDINGS:` names the highest one. A not-ready reply with no readable severity still - prints `FINDINGS: unspecified` and exits 3. + prints `FINDINGS: unspecified` and exits 3. Any other last `Verdict:` + (`Can't verify`, `blocked`, `revise`, an empty one), or a `ready` one that + says something could not be verified, is + [`unverified`](#outside-review-unverified-verdict), never `clean`. +- **Numbered findings.** In a reply with a `Verdict:` line, every numbered + item at the start of a line (`1. x`, `1. **Nonblocking — x**`) is a finding. + When one has no readable severity, `FINDINGS:` prints `unspecified` instead + of `none`; the verdict still comes from the rules above. - **No findings.** A review with no tag and no label is `clean` only when its conclusion says so: the last sentence (or its last ", and" clause), or the `because` clause of its `Recommendation:` line, opens with `NO_FINDINGS`, @@ -380,6 +387,26 @@ bun lib/outside-review-result.ts --verdict execution reply.md **Expected result.** `VERDICT: findings` or `VERDICT: clean` with a P0-P3 `FINDINGS:` value, and no `UNPARSED:` line. + +### `... outside review: ran, verdict unverified (the reviewer's own verdict does not say the change is ready (Verdict: ...))` / `REASON: unverified_verdict` + +**Meaning.** The reply's last `Verdict:` line is neither `ready` nor +`not ready`, for example `Verdict: Can't verify (one changed file excluded +...)`, or it says `ready` but also that something could not be verified. The +reviewer did not say the review finished and found the change ready, so gstack +does not count it as `clean`. The quoted verdict is in parentheses. + +**What is kept.** The full reply is shown above. + +**Fix.** Read the verdict and the reply, and decide; it is not a pass. Fix +what the verdict names (a file left out of the review, a suite that could not +run), then re-run the review. To re-check a saved reply, run +`bun lib/outside-review-result.ts --verdict execution reply.md` from your +gstack checkout. + +**Expected result.** `VERDICT: clean` or `VERDICT: findings` with a last +`Verdict: ready` or `Verdict: not ready` line. + ### `CODEX_MODE: model_unusable` / `MODEL_UNUSABLE` diff --git a/lib/gate-outcomes.ts b/lib/gate-outcomes.ts index d29ade2be..f816d71a2 100644 --- a/lib/gate-outcomes.ts +++ b/lib/gate-outcomes.ts @@ -68,6 +68,11 @@ export const GATE_OUTCOMES = { summary: 'the review labels a finding with a P0-P3 severity in a form the parser cannot read', fix: 'read the quoted line and the full reply above and decide; it is not a pass. Re-run the review asking for [P0]-[P3] or "P1:" labels, or re-check a corrected reply with bun lib/outside-review-result.ts --verdict from your gstack checkout (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unparsed-severity)', }, + unverified_verdict: { + state: 'ran', anchor: 'outside-review-unverified-verdict', + summary: "the reviewer's own verdict does not say the change is ready", + fix: 'read the quoted verdict and the full reply above and decide; it is not a pass. Fix what the verdict names (an excluded file, a suite that could not run), then re-run the review (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unverified-verdict)', + }, model_unusable: { state: 'unavailable', anchor: 'codex-model-unusable', summary: 'Codex could not use the selected model', diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index 397c94ba2..200206ee1 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -24,7 +24,8 @@ export interface OutsideReviewInput { export interface OutsideReviewClassification { execution: { state: 'ran' } | { state: 'unavailable'; reason: GateReason; detail?: string }; - findings: { highest: Severity | null }; + /** unlabeled: numbered findings with no readable severity, in a reply with a `Verdict:` line; set only when non-zero. */ + findings: { highest: Severity | null; unlabeled?: number }; verdict: OutsideVerdict; /** Set for every verdict except clean/findings. */ reason?: GateReason; @@ -249,11 +250,14 @@ function firstUnparsed(text: string): string | undefined { return at < 0 ? undefined : `line ${at + 1}: "${lines[at]!.trim().slice(0, 160)}"`; } -/** The reply's last `Verdict:` line says it is not ready. */ -function verdictNotReady(plain: string): boolean { - const last = [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1]; - return last !== undefined && /^not[\t ]+ready\b/i.test(last.trim()); +/** The value of the reply's last `Verdict:` line, or undefined when it has none. */ +function lastVerdict(plain: string): string | undefined { + return [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1]!.trim(); } +/** A verdict that says the reviewer could not check something is never a ready one (#3925). */ +const UNVERIFIED_CLAIM = /\b(?:can['\u2019]?t|cannot|can not|couldn['\u2019]t|could not|unable to|not able to)\s+(?:be\s+)?verif|\bunverifi|\bnot (?:been )?verified\b/i; +/** A numbered item at the start of a line; in a reply with a `Verdict:` line it is a finding (#3925). */ +const NUMBERED_ITEM = /^(?:\*\*|__)?\(?\d{1,3}[.)][\t ]+\S.*$/gm; export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification { const plain = plainReview(input.text); @@ -263,7 +267,10 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC const levelsOf = (c: string, p: string) => [...[...p.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity), ...severityTokens(c), ...(reviewed ? severityWords(c) : [])]; const levels = levelsOf(read, plainRead); - const findings = { highest: levels.length ? levels.sort()[0]! : null }; + const verdicts = [lastVerdict(plainCode), lastVerdict(plainRead)]; + const unlabeled = reviewed && verdicts[1] !== undefined + ? [...read.matchAll(NUMBERED_ITEM)].filter(m => !levelsOf(m[0], plainReview(m[0])).length).length : 0; + const findings = { highest: levels.length ? levels.sort()[0]! : null, ...(unlabeled ? { unlabeled } : {}) }; const cleanEvidence = levelsOf(code, plainCode).length > 0; const ran = execution(input); if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail }; @@ -283,9 +290,11 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC if (!reviewed) return result(blocking ? 'findings' : 'clean'); const unparsed = firstUnparsed(read); // A last explicit not-ready blocks even when the only parsed severities are advisory. - const notReady = verdictNotReady(plainCode) || verdictNotReady(plainRead); + const notReady = verdicts.some(v => v !== undefined && /^not[\t ]+ready\b/i.test(v)); if (blocking || notReady) return { ...result('findings'), ...(unparsed ? { unparsed } : {}) }; if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed }; + const unready = verdicts.find(v => v !== undefined && (!/^ready\b/i.test(v) || UNVERIFIED_CLAIM.test(v))); + if (unready !== undefined) return result('unverified', 'unverified_verdict', `Verdict: ${unready}`.trim().slice(0, 160)); if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(code)) { return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion'); } @@ -388,7 +397,7 @@ if (import.meta.main) { const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events }); const label = flags['--label'] ?? 'Outside review'; console.log(`VERDICT: ${checked.verdict}`); - console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`); + console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' || checked.findings.unlabeled ? 'unspecified' : 'none')}`); if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`); if (flags['--events'] !== undefined) console.log(formatUsageLine(usageFromEvents(events))); if (checked.reason) { diff --git a/test/fixtures/outside-review-severity/codex-3925/README.md b/test/fixtures/outside-review-severity/codex-3925/README.md new file mode 100644 index 000000000..3788bcffd --- /dev/null +++ b/test/fixtures/outside-review-severity/codex-3925/README.md @@ -0,0 +1,15 @@ +# Codex reply shapes from #3925 + +Issue #3925 reports two Codex consult replies that the outside-review helper +summarised as `VERDICT: clean / FINDINGS: none`. The issue describes them but +does not include them, so these files are synthesized from what it quotes: + +- `cant-verify.txt`: a reply whose last line is `Verdict: Can't verify (...)`. + The reviewer says it could not finish, so it is not `clean`. +- `nonblocking-numbered.txt`: one numbered finding labeled `Nonblocking`, no + P0-P3 tag, then `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)`. + The verdict is `clean`, but `FINDINGS:` must not say `none`. +- `ready-clean.txt`: the negative control, a real clean reply with + `Verdict: ready` and no findings. + +`test/outside-review-unparsed-severity.test.ts` holds the expected verdicts. diff --git a/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt b/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt new file mode 100644 index 000000000..55982cf43 --- /dev/null +++ b/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt @@ -0,0 +1,5 @@ +I read the diff for the files I could open. `lib/sync.ts` was excluded from +the review scope, and the four required suites could not start in the sandbox, +so I could not run them. + +Verdict: Can't verify (one changed file excluded and four required suites blocked) diff --git a/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt b/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt new file mode 100644 index 000000000..fcf4c7b79 --- /dev/null +++ b/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt @@ -0,0 +1,6 @@ +I read the diff and ran `bun test test/retry.test.ts`. + +1. **Nonblocking — The retry log line repeats the attempt count.** `lib/retry.ts:41` + +Verdict: ready +END OF REPORT: 1 findings (0 blocking) diff --git a/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt b/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt new file mode 100644 index 000000000..0aa1b73ed --- /dev/null +++ b/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt @@ -0,0 +1,5 @@ +I read the diff, ran `bun test test/retry.test.ts`, and checked every caller of +`retry()`. The cap is enforced and the tests cover the boundary. + +Verdict: ready +END OF REPORT: 0 findings (0 blocking) diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 1cc026c28..5dff52d1d 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -176,7 +176,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-margin-elided-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-packet-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], - 'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts'], + 'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts', 'test/outside-review-unparsed-severity.test.ts'], 'test/fixtures/outside-voice-failure-prose-allowlist.json': ['test/outside-voice-failure-policy.test.ts'], 'test/fixtures/overlay-v4-replay-37198445662.json': ['test/overlay-measurement.test.ts'], 'test/fixtures/paid-report-census/**': ['test/paid-report-census.test.ts', 'test/paid-shards.test.ts'], diff --git a/test/outside-review-unparsed-severity.test.ts b/test/outside-review-unparsed-severity.test.ts index 82e7d0f1a..34cee204c 100644 --- a/test/outside-review-unparsed-severity.test.ts +++ b/test/outside-review-unparsed-severity.test.ts @@ -7,6 +7,9 @@ * forms the issue documents (it describes 13 replies but does not include them). * #3329: that same gate then printed `clean` for a parsed advisory finding * whose last `Verdict:` line says `not ready`. `bare-dash-p2.txt` is unchanged. + * #3925: it printed `clean` for a `Verdict: Can't verify` reply, and + * `FINDINGS: none` for a numbered finding labeled `Nonblocking` with no P-tag + * (fixtures in test/fixtures/outside-review-severity/codex-3925/). */ import { describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; @@ -184,3 +187,62 @@ describe('#3329 a last not-ready Verdict blocks even with advisory findings', () expect([failing.verdict, failing.score]).toEqual(['findings', 6]); }); }); + +describe('#3925 an unrecognized verdict is not clean; numbered findings never print FINDINGS: none', () => { + const FIX3925 = path.join(ROOT, 'test', 'fixtures', 'outside-review-severity', 'codex-3925'); + const cli3925 = (gate: OutsideGate, name: string) => + spawnSync(process.execPath, [LIB, '--verdict', '--label', 'Codex consult', '--exit', '0', gate, path.join(FIX3925, name)], { encoding: 'utf8', timeout: 10000 }); + const textFor = (gate: OutsideGate, text: string) => gate === 'review' ? `${text}\n${REC}` : text; + const see = (text: string, gate: OutsideGate) => { + const r = classifyOutsideReview({ text: textFor(gate, text), gate }); + return { verdict: r.verdict as string, reason: (r.reason ?? null) as string | null, unlabeled: r.findings.unlabeled ?? 0 }; + }; + + for (const [name, status, stdout] of [ + ['cant-verify.txt', 4, 'VERDICT: unverified\nFINDINGS: none\nREASON: unverified_verdict\n'], + ['nonblocking-numbered.txt', 0, 'VERDICT: clean\nFINDINGS: unspecified\n'], + ['ready-clean.txt', 0, 'VERDICT: clean\nFINDINGS: none\n'], + ] as const) { + test(`${name}: exit ${status}`, () => { + const r = cli3925('execution', name); + expect([r.status, r.stdout]).toEqual([status, stdout]); + }); + } + + test('a cannot-verify verdict prints its reason, the quoted verdict and the anchor', () => { + const r = cli3925('execution', 'cant-verify.txt'); + expect(r.stderr).toContain('Codex consult: ran, verdict unverified ('); + expect(r.stderr).toContain("Verdict: Can't verify (one changed file excluded"); + expect(r.stderr).toContain('docs/troubleshooting.md#outside-review-unverified-verdict'); + expect(GATE_OUTCOMES.unverified_verdict).toMatchObject({ state: 'ran', anchor: 'outside-review-unverified-verdict' }); + expect(validateOutsideReview(read('no-findings.txt').replace('Verdict: ready', "Verdict: Can't verify"), 'execution').completed).toBe(false); + }); + + test('every verdict other than ready, and a ready verdict that says it could not verify, is unverified on every review gate', () => { + for (const gate of ['execution', 'structured', 'review'] as const) + for (const verdictLine of [ + "Verdict: Can't verify (suites blocked)", 'Verdict: cannot verify', 'Verdict: Unable to verify the Windows path', + "**Verdict:** Can\u2019t verify", '### Verdict: blocked', 'Verdict: revise', 'Verdict:', + 'Verdict: ready, but I could not verify the migration', 'Verdict: ready (unverified)', + ]) + expect({ gate, verdictLine, ...see(`No issues found.\n${verdictLine}`, gate) }).toEqual({ gate, verdictLine, verdict: 'unverified', reason: 'unverified_verdict', unlabeled: 0 }); + }); + + test('numbered findings without a P-tag are counted, whatever their label', () => { + for (const label of ['**Nonblocking — The log repeats.**', 'Nonblocking: the log repeats.', '**Minor** — the log repeats.', 'The log repeats the attempt count.']) + expect({ label, ...see(`1. ${label}\nVerdict: ready`, 'execution') }).toEqual({ label, verdict: 'clean', reason: null, unlabeled: 1 }); + expect(see('1. **P3 — Rename it.**\n2. Nonblocking — the log repeats.\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 1 }); + expect(classifyOutsideReview({ text: fs.readFileSync(path.join(FIX3925, 'nonblocking-numbered.txt'), 'utf8'), gate: 'structured' })).toMatchObject({ verdict: 'unverified', reason: 'untagged_review', findings: { highest: null, unlabeled: 1 } }); + }); + + test('negative controls: real clean replies, consult answers, tagged findings and quoted examples stay as they were', () => { + for (const gate of ['execution', 'structured', 'review'] as const) { + expect({ gate, ...see(read('no-findings.txt'), gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 }); + expect({ gate, ...see('No issues found.\nVerdict: Ready to merge.', gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 }); + } + expect(see('1. Use a queue.\n2. Add a retry cap.', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 }); + expect(see('1. **P2 — The docs lag.**\n2. **P3 — Rename it.**\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 }); + expect(see('No issues found.\n```\n1. Nonblocking — example\nVerdict: cannot verify\n```\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 }); + expect(see('1. The retry path can double-send.\nVerdict: not ready', 'execution')).toEqual({ verdict: 'findings', reason: null, unlabeled: 1 }); + }); +});