diff --git a/changelog.d/gsta259-3925-unverified-verdict.md b/changelog.d/gsta259-3925-unverified-verdict.md
new file mode 100644
index 000000000..9e70c40b5
--- /dev/null
+++ b/changelog.d/gsta259-3925-unverified-verdict.md
@@ -0,0 +1,4 @@
+### Fixed
+
+- **An outside review that says it could not verify the change is no longer read as a pass (#3925).** `lib/outside-review-result.ts` printed `VERDICT: clean` / `FINDINGS: none` for a Codex reply ending `Verdict: Can't verify (one changed file excluded and four required suites blocked)`. A last `Verdict:` line now has to say `ready` (or `not ready`, which still blocks) for the reply to count; anything else, and a `ready` that says something could not be verified, is `VERDICT: unverified` with the new `REASON: unverified_verdict` and exit 4. See docs/troubleshooting.md#outside-review-unverified-verdict.
+- **A numbered finding without a P0-P3 tag is no longer dropped (#3925).** A reply with `1. **Nonblocking — x**`, `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)` printed `FINDINGS: none`. In a reply with a `Verdict:` line, every numbered item is now counted as a finding, and one with no readable severity makes `FINDINGS:` print `unspecified`. The verdict is unchanged (`clean` here, since the reviewer said ready). A consult answer with no `Verdict:` line, and a real clean reply, still read `clean` / `none`.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index f17f55959..9beeefa94 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -179,7 +179,14 @@ the readiness dashboard and the PR body, and never count it as a pass.
A reply whose last `Verdict:` line says `not ready` is `findings` and exits
3 even when readable advisory severities (P2 or P3) were parsed; `FINDINGS:`
names the highest one. A not-ready reply with no readable severity still
- prints `FINDINGS: unspecified` and exits 3.
+ prints `FINDINGS: unspecified` and exits 3. Any other last `Verdict:`
+ (`Can't verify`, `blocked`, `revise`, an empty one), or a `ready` one that
+ says something could not be verified, is
+ [`unverified`](#outside-review-unverified-verdict), never `clean`.
+- **Numbered findings.** In a reply with a `Verdict:` line, every numbered
+ item at the start of a line (`1. x`, `1. **Nonblocking — x**`) is a finding.
+ When one has no readable severity, `FINDINGS:` prints `unspecified` instead
+ of `none`; the verdict still comes from the rules above.
- **No findings.** A review with no tag and no label is `clean` only when its
conclusion says so: the last sentence (or its last ", and" clause), or the
`because` clause of its `Recommendation:` line, opens with `NO_FINDINGS`,
@@ -380,6 +387,26 @@ bun lib/outside-review-result.ts --verdict execution reply.md
**Expected result.** `VERDICT: findings` or `VERDICT: clean` with a P0-P3
`FINDINGS:` value, and no `UNPARSED:` line.
+
+### `... outside review: ran, verdict unverified (the reviewer's own verdict does not say the change is ready (Verdict: ...))` / `REASON: unverified_verdict`
+
+**Meaning.** The reply's last `Verdict:` line is neither `ready` nor
+`not ready`, for example `Verdict: Can't verify (one changed file excluded
+...)`, or it says `ready` but also that something could not be verified. The
+reviewer did not say the review finished and found the change ready, so gstack
+does not count it as `clean`. The quoted verdict is in parentheses.
+
+**What is kept.** The full reply is shown above.
+
+**Fix.** Read the verdict and the reply, and decide; it is not a pass. Fix
+what the verdict names (a file left out of the review, a suite that could not
+run), then re-run the review. To re-check a saved reply, run
+`bun lib/outside-review-result.ts --verdict execution reply.md` from your
+gstack checkout.
+
+**Expected result.** `VERDICT: clean` or `VERDICT: findings` with a last
+`Verdict: ready` or `Verdict: not ready` line.
+
### `CODEX_MODE: model_unusable` / `MODEL_UNUSABLE`
diff --git a/lib/gate-outcomes.ts b/lib/gate-outcomes.ts
index d29ade2be..f816d71a2 100644
--- a/lib/gate-outcomes.ts
+++ b/lib/gate-outcomes.ts
@@ -68,6 +68,11 @@ export const GATE_OUTCOMES = {
summary: 'the review labels a finding with a P0-P3 severity in a form the parser cannot read',
fix: 'read the quoted line and the full reply above and decide; it is not a pass. Re-run the review asking for [P0]-[P3] or "P1:" labels, or re-check a corrected reply with bun lib/outside-review-result.ts --verdict from your gstack checkout (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unparsed-severity)',
},
+ unverified_verdict: {
+ state: 'ran', anchor: 'outside-review-unverified-verdict',
+ summary: "the reviewer's own verdict does not say the change is ready",
+ fix: 'read the quoted verdict and the full reply above and decide; it is not a pass. Fix what the verdict names (an excluded file, a suite that could not run), then re-run the review (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unverified-verdict)',
+ },
model_unusable: {
state: 'unavailable', anchor: 'codex-model-unusable',
summary: 'Codex could not use the selected model',
diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts
index 397c94ba2..200206ee1 100644
--- a/lib/outside-review-result.ts
+++ b/lib/outside-review-result.ts
@@ -24,7 +24,8 @@ export interface OutsideReviewInput {
export interface OutsideReviewClassification {
execution: { state: 'ran' } | { state: 'unavailable'; reason: GateReason; detail?: string };
- findings: { highest: Severity | null };
+ /** unlabeled: numbered findings with no readable severity, in a reply with a `Verdict:` line; set only when non-zero. */
+ findings: { highest: Severity | null; unlabeled?: number };
verdict: OutsideVerdict;
/** Set for every verdict except clean/findings. */
reason?: GateReason;
@@ -249,11 +250,14 @@ function firstUnparsed(text: string): string | undefined {
return at < 0 ? undefined : `line ${at + 1}: "${lines[at]!.trim().slice(0, 160)}"`;
}
-/** The reply's last `Verdict:` line says it is not ready. */
-function verdictNotReady(plain: string): boolean {
- const last = [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1];
- return last !== undefined && /^not[\t ]+ready\b/i.test(last.trim());
+/** The value of the reply's last `Verdict:` line, or undefined when it has none. */
+function lastVerdict(plain: string): string | undefined {
+ return [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1]!.trim();
}
+/** A verdict that says the reviewer could not check something is never a ready one (#3925). */
+const UNVERIFIED_CLAIM = /\b(?:can['\u2019]?t|cannot|can not|couldn['\u2019]t|could not|unable to|not able to)\s+(?:be\s+)?verif|\bunverifi|\bnot (?:been )?verified\b/i;
+/** A numbered item at the start of a line; in a reply with a `Verdict:` line it is a finding (#3925). */
+const NUMBERED_ITEM = /^(?:\*\*|__)?\(?\d{1,3}[.)][\t ]+\S.*$/gm;
export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification {
const plain = plainReview(input.text);
@@ -263,7 +267,10 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
const levelsOf = (c: string, p: string) => [...[...p.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity),
...severityTokens(c), ...(reviewed ? severityWords(c) : [])];
const levels = levelsOf(read, plainRead);
- const findings = { highest: levels.length ? levels.sort()[0]! : null };
+ const verdicts = [lastVerdict(plainCode), lastVerdict(plainRead)];
+ const unlabeled = reviewed && verdicts[1] !== undefined
+ ? [...read.matchAll(NUMBERED_ITEM)].filter(m => !levelsOf(m[0], plainReview(m[0])).length).length : 0;
+ const findings = { highest: levels.length ? levels.sort()[0]! : null, ...(unlabeled ? { unlabeled } : {}) };
const cleanEvidence = levelsOf(code, plainCode).length > 0;
const ran = execution(input);
if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail };
@@ -283,9 +290,11 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
if (!reviewed) return result(blocking ? 'findings' : 'clean');
const unparsed = firstUnparsed(read);
// A last explicit not-ready blocks even when the only parsed severities are advisory.
- const notReady = verdictNotReady(plainCode) || verdictNotReady(plainRead);
+ const notReady = verdicts.some(v => v !== undefined && /^not[\t ]+ready\b/i.test(v));
if (blocking || notReady) return { ...result('findings'), ...(unparsed ? { unparsed } : {}) };
if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed };
+ const unready = verdicts.find(v => v !== undefined && (!/^ready\b/i.test(v) || UNVERIFIED_CLAIM.test(v)));
+ if (unready !== undefined) return result('unverified', 'unverified_verdict', `Verdict: ${unready}`.trim().slice(0, 160));
if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(code)) {
return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
}
@@ -388,7 +397,7 @@ if (import.meta.main) {
const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events });
const label = flags['--label'] ?? 'Outside review';
console.log(`VERDICT: ${checked.verdict}`);
- console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`);
+ console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' || checked.findings.unlabeled ? 'unspecified' : 'none')}`);
if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`);
if (flags['--events'] !== undefined) console.log(formatUsageLine(usageFromEvents(events)));
if (checked.reason) {
diff --git a/test/fixtures/outside-review-severity/codex-3925/README.md b/test/fixtures/outside-review-severity/codex-3925/README.md
new file mode 100644
index 000000000..3788bcffd
--- /dev/null
+++ b/test/fixtures/outside-review-severity/codex-3925/README.md
@@ -0,0 +1,15 @@
+# Codex reply shapes from #3925
+
+Issue #3925 reports two Codex consult replies that the outside-review helper
+summarised as `VERDICT: clean / FINDINGS: none`. The issue describes them but
+does not include them, so these files are synthesized from what it quotes:
+
+- `cant-verify.txt`: a reply whose last line is `Verdict: Can't verify (...)`.
+ The reviewer says it could not finish, so it is not `clean`.
+- `nonblocking-numbered.txt`: one numbered finding labeled `Nonblocking`, no
+ P0-P3 tag, then `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)`.
+ The verdict is `clean`, but `FINDINGS:` must not say `none`.
+- `ready-clean.txt`: the negative control, a real clean reply with
+ `Verdict: ready` and no findings.
+
+`test/outside-review-unparsed-severity.test.ts` holds the expected verdicts.
diff --git a/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt b/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt
new file mode 100644
index 000000000..55982cf43
--- /dev/null
+++ b/test/fixtures/outside-review-severity/codex-3925/cant-verify.txt
@@ -0,0 +1,5 @@
+I read the diff for the files I could open. `lib/sync.ts` was excluded from
+the review scope, and the four required suites could not start in the sandbox,
+so I could not run them.
+
+Verdict: Can't verify (one changed file excluded and four required suites blocked)
diff --git a/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt b/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt
new file mode 100644
index 000000000..fcf4c7b79
--- /dev/null
+++ b/test/fixtures/outside-review-severity/codex-3925/nonblocking-numbered.txt
@@ -0,0 +1,6 @@
+I read the diff and ran `bun test test/retry.test.ts`.
+
+1. **Nonblocking — The retry log line repeats the attempt count.** `lib/retry.ts:41`
+
+Verdict: ready
+END OF REPORT: 1 findings (0 blocking)
diff --git a/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt b/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt
new file mode 100644
index 000000000..0aa1b73ed
--- /dev/null
+++ b/test/fixtures/outside-review-severity/codex-3925/ready-clean.txt
@@ -0,0 +1,5 @@
+I read the diff, ran `bun test test/retry.test.ts`, and checked every caller of
+`retry()`. The cap is enforced and the tests cover the boundary.
+
+Verdict: ready
+END OF REPORT: 0 findings (0 blocking)
diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts
index 1cc026c28..5dff52d1d 100644
--- a/test/helpers/free-fixtures-data.ts
+++ b/test/helpers/free-fixtures-data.ts
@@ -176,7 +176,7 @@ export const FREE_FIXTURES: Readonly> = {
'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
'test/fixtures/native-auq-margin-elided-sep21.json': ['test/auq-native-capture.test.ts'],
'test/fixtures/native-auq-packet-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
- 'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts'],
+ 'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts', 'test/outside-review-unparsed-severity.test.ts'],
'test/fixtures/outside-voice-failure-prose-allowlist.json': ['test/outside-voice-failure-policy.test.ts'],
'test/fixtures/overlay-v4-replay-37198445662.json': ['test/overlay-measurement.test.ts'],
'test/fixtures/paid-report-census/**': ['test/paid-report-census.test.ts', 'test/paid-shards.test.ts'],
diff --git a/test/outside-review-unparsed-severity.test.ts b/test/outside-review-unparsed-severity.test.ts
index 82e7d0f1a..34cee204c 100644
--- a/test/outside-review-unparsed-severity.test.ts
+++ b/test/outside-review-unparsed-severity.test.ts
@@ -7,6 +7,9 @@
* forms the issue documents (it describes 13 replies but does not include them).
* #3329: that same gate then printed `clean` for a parsed advisory finding
* whose last `Verdict:` line says `not ready`. `bare-dash-p2.txt` is unchanged.
+ * #3925: it printed `clean` for a `Verdict: Can't verify` reply, and
+ * `FINDINGS: none` for a numbered finding labeled `Nonblocking` with no P-tag
+ * (fixtures in test/fixtures/outside-review-severity/codex-3925/).
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
@@ -184,3 +187,62 @@ describe('#3329 a last not-ready Verdict blocks even with advisory findings', ()
expect([failing.verdict, failing.score]).toEqual(['findings', 6]);
});
});
+
+describe('#3925 an unrecognized verdict is not clean; numbered findings never print FINDINGS: none', () => {
+ const FIX3925 = path.join(ROOT, 'test', 'fixtures', 'outside-review-severity', 'codex-3925');
+ const cli3925 = (gate: OutsideGate, name: string) =>
+ spawnSync(process.execPath, [LIB, '--verdict', '--label', 'Codex consult', '--exit', '0', gate, path.join(FIX3925, name)], { encoding: 'utf8', timeout: 10000 });
+ const textFor = (gate: OutsideGate, text: string) => gate === 'review' ? `${text}\n${REC}` : text;
+ const see = (text: string, gate: OutsideGate) => {
+ const r = classifyOutsideReview({ text: textFor(gate, text), gate });
+ return { verdict: r.verdict as string, reason: (r.reason ?? null) as string | null, unlabeled: r.findings.unlabeled ?? 0 };
+ };
+
+ for (const [name, status, stdout] of [
+ ['cant-verify.txt', 4, 'VERDICT: unverified\nFINDINGS: none\nREASON: unverified_verdict\n'],
+ ['nonblocking-numbered.txt', 0, 'VERDICT: clean\nFINDINGS: unspecified\n'],
+ ['ready-clean.txt', 0, 'VERDICT: clean\nFINDINGS: none\n'],
+ ] as const) {
+ test(`${name}: exit ${status}`, () => {
+ const r = cli3925('execution', name);
+ expect([r.status, r.stdout]).toEqual([status, stdout]);
+ });
+ }
+
+ test('a cannot-verify verdict prints its reason, the quoted verdict and the anchor', () => {
+ const r = cli3925('execution', 'cant-verify.txt');
+ expect(r.stderr).toContain('Codex consult: ran, verdict unverified (');
+ expect(r.stderr).toContain("Verdict: Can't verify (one changed file excluded");
+ expect(r.stderr).toContain('docs/troubleshooting.md#outside-review-unverified-verdict');
+ expect(GATE_OUTCOMES.unverified_verdict).toMatchObject({ state: 'ran', anchor: 'outside-review-unverified-verdict' });
+ expect(validateOutsideReview(read('no-findings.txt').replace('Verdict: ready', "Verdict: Can't verify"), 'execution').completed).toBe(false);
+ });
+
+ test('every verdict other than ready, and a ready verdict that says it could not verify, is unverified on every review gate', () => {
+ for (const gate of ['execution', 'structured', 'review'] as const)
+ for (const verdictLine of [
+ "Verdict: Can't verify (suites blocked)", 'Verdict: cannot verify', 'Verdict: Unable to verify the Windows path',
+ "**Verdict:** Can\u2019t verify", '### Verdict: blocked', 'Verdict: revise', 'Verdict:',
+ 'Verdict: ready, but I could not verify the migration', 'Verdict: ready (unverified)',
+ ])
+ expect({ gate, verdictLine, ...see(`No issues found.\n${verdictLine}`, gate) }).toEqual({ gate, verdictLine, verdict: 'unverified', reason: 'unverified_verdict', unlabeled: 0 });
+ });
+
+ test('numbered findings without a P-tag are counted, whatever their label', () => {
+ for (const label of ['**Nonblocking — The log repeats.**', 'Nonblocking: the log repeats.', '**Minor** — the log repeats.', 'The log repeats the attempt count.'])
+ expect({ label, ...see(`1. ${label}\nVerdict: ready`, 'execution') }).toEqual({ label, verdict: 'clean', reason: null, unlabeled: 1 });
+ expect(see('1. **P3 — Rename it.**\n2. Nonblocking — the log repeats.\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 1 });
+ expect(classifyOutsideReview({ text: fs.readFileSync(path.join(FIX3925, 'nonblocking-numbered.txt'), 'utf8'), gate: 'structured' })).toMatchObject({ verdict: 'unverified', reason: 'untagged_review', findings: { highest: null, unlabeled: 1 } });
+ });
+
+ test('negative controls: real clean replies, consult answers, tagged findings and quoted examples stay as they were', () => {
+ for (const gate of ['execution', 'structured', 'review'] as const) {
+ expect({ gate, ...see(read('no-findings.txt'), gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 });
+ expect({ gate, ...see('No issues found.\nVerdict: Ready to merge.', gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 });
+ }
+ expect(see('1. Use a queue.\n2. Add a retry cap.', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
+ expect(see('1. **P2 — The docs lag.**\n2. **P3 — Rename it.**\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
+ expect(see('No issues found.\n```\n1. Nonblocking — example\nVerdict: cannot verify\n```\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
+ expect(see('1. The retry path can double-send.\nVerdict: not ready', 'execution')).toEqual({ verdict: 'findings', reason: null, unlabeled: 1 });
+ });
+});