mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 07:04:17 +01:00
fix(outside-review): an unrecognized Verdict is unverified, numbered findings are counted (#3925)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2f9d11a59d
commit
7e72adf6b6
10 files changed
+148
-10
No files matched your search
@@ -0,0 +1,4 @@
|
||||
### Fixed
|
||||
|
||||
- **An outside review that says it could not verify the change is no longer read as a pass (#3925).** `lib/outside-review-result.ts` printed `VERDICT: clean` / `FINDINGS: none` for a Codex reply ending `Verdict: Can't verify (one changed file excluded and four required suites blocked)`. A last `Verdict:` line now has to say `ready` (or `not ready`, which still blocks) for the reply to count; anything else, and a `ready` that says something could not be verified, is `VERDICT: unverified` with the new `REASON: unverified_verdict` and exit 4. See docs/troubleshooting.md#outside-review-unverified-verdict.
|
||||
- **A numbered finding without a P0-P3 tag is no longer dropped (#3925).** A reply with `1. **Nonblocking — x**`, `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)` printed `FINDINGS: none`. In a reply with a `Verdict:` line, every numbered item is now counted as a finding, and one with no readable severity makes `FINDINGS:` print `unspecified`. The verdict is unchanged (`clean` here, since the reviewer said ready). A consult answer with no `Verdict:` line, and a real clean reply, still read `clean` / `none`.
|
||||
+28
-1
@@ -179,7 +179,14 @@ the readiness dashboard and the PR body, and never count it as a pass.
|
||||
A reply whose last `Verdict:` line says `not ready` is `findings` and exits
|
||||
3 even when readable advisory severities (P2 or P3) were parsed; `FINDINGS:`
|
||||
names the highest one. A not-ready reply with no readable severity still
|
||||
prints `FINDINGS: unspecified` and exits 3.
|
||||
prints `FINDINGS: unspecified` and exits 3. Any other last `Verdict:`
|
||||
(`Can't verify`, `blocked`, `revise`, an empty one), or a `ready` one that
|
||||
says something could not be verified, is
|
||||
[`unverified`](#outside-review-unverified-verdict), never `clean`.
|
||||
- **Numbered findings.** In a reply with a `Verdict:` line, every numbered
|
||||
item at the start of a line (`1. x`, `1. **Nonblocking — x**`) is a finding.
|
||||
When one has no readable severity, `FINDINGS:` prints `unspecified` instead
|
||||
of `none`; the verdict still comes from the rules above.
|
||||
- **No findings.** A review with no tag and no label is `clean` only when its
|
||||
conclusion says so: the last sentence (or its last ", and" clause), or the
|
||||
`because` clause of its `Recommendation:` line, opens with `NO_FINDINGS`,
|
||||
@@ -380,6 +387,26 @@ bun lib/outside-review-result.ts --verdict execution reply.md
|
||||
**Expected result.** `VERDICT: findings` or `VERDICT: clean` with a P0-P3
|
||||
`FINDINGS:` value, and no `UNPARSED:` line.
|
||||
|
||||
<a id="outside-review-unverified-verdict"></a>
|
||||
### `... outside review: ran, verdict unverified (the reviewer's own verdict does not say the change is ready (Verdict: ...))` / `REASON: unverified_verdict`
|
||||
|
||||
**Meaning.** The reply's last `Verdict:` line is neither `ready` nor
|
||||
`not ready`, for example `Verdict: Can't verify (one changed file excluded
|
||||
...)`, or it says `ready` but also that something could not be verified. The
|
||||
reviewer did not say the review finished and found the change ready, so gstack
|
||||
does not count it as `clean`. The quoted verdict is in parentheses.
|
||||
|
||||
**What is kept.** The full reply is shown above.
|
||||
|
||||
**Fix.** Read the verdict and the reply, and decide; it is not a pass. Fix
|
||||
what the verdict names (a file left out of the review, a suite that could not
|
||||
run), then re-run the review. To re-check a saved reply, run
|
||||
`bun lib/outside-review-result.ts --verdict execution reply.md` from your
|
||||
gstack checkout.
|
||||
|
||||
**Expected result.** `VERDICT: clean` or `VERDICT: findings` with a last
|
||||
`Verdict: ready` or `Verdict: not ready` line.
|
||||
|
||||
<a id="codex-model-unusable"></a>
|
||||
### `CODEX_MODE: model_unusable` / `MODEL_UNUSABLE`
|
||||
|
||||
|
||||
@@ -68,6 +68,11 @@ export const GATE_OUTCOMES = {
|
||||
summary: 'the review labels a finding with a P0-P3 severity in a form the parser cannot read',
|
||||
fix: 'read the quoted line and the full reply above and decide; it is not a pass. Re-run the review asking for [P0]-[P3] or "P1:" labels, or re-check a corrected reply with bun lib/outside-review-result.ts --verdict <gate> <reply-file> from your gstack checkout (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unparsed-severity)',
|
||||
},
|
||||
unverified_verdict: {
|
||||
state: 'ran', anchor: 'outside-review-unverified-verdict',
|
||||
summary: "the reviewer's own verdict does not say the change is ready",
|
||||
fix: 'read the quoted verdict and the full reply above and decide; it is not a pass. Fix what the verdict names (an excluded file, a suite that could not run), then re-run the review (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-unverified-verdict)',
|
||||
},
|
||||
model_unusable: {
|
||||
state: 'unavailable', anchor: 'codex-model-unusable',
|
||||
summary: 'Codex could not use the selected model',
|
||||
|
||||
@@ -24,7 +24,8 @@ export interface OutsideReviewInput {
|
||||
|
||||
export interface OutsideReviewClassification {
|
||||
execution: { state: 'ran' } | { state: 'unavailable'; reason: GateReason; detail?: string };
|
||||
findings: { highest: Severity | null };
|
||||
/** unlabeled: numbered findings with no readable severity, in a reply with a `Verdict:` line; set only when non-zero. */
|
||||
findings: { highest: Severity | null; unlabeled?: number };
|
||||
verdict: OutsideVerdict;
|
||||
/** Set for every verdict except clean/findings. */
|
||||
reason?: GateReason;
|
||||
@@ -249,11 +250,14 @@ function firstUnparsed(text: string): string | undefined {
|
||||
return at < 0 ? undefined : `line ${at + 1}: "${lines[at]!.trim().slice(0, 160)}"`;
|
||||
}
|
||||
|
||||
/** The reply's last `Verdict:` line says it is not ready. */
|
||||
function verdictNotReady(plain: string): boolean {
|
||||
const last = [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1];
|
||||
return last !== undefined && /^not[\t ]+ready\b/i.test(last.trim());
|
||||
/** The value of the reply's last `Verdict:` line, or undefined when it has none. */
|
||||
function lastVerdict(plain: string): string | undefined {
|
||||
return [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1]!.trim();
|
||||
}
|
||||
/** A verdict that says the reviewer could not check something is never a ready one (#3925). */
|
||||
const UNVERIFIED_CLAIM = /\b(?:can['\u2019]?t|cannot|can not|couldn['\u2019]t|could not|unable to|not able to)\s+(?:be\s+)?verif|\bunverifi|\bnot (?:been )?verified\b/i;
|
||||
/** A numbered item at the start of a line; in a reply with a `Verdict:` line it is a finding (#3925). */
|
||||
const NUMBERED_ITEM = /^(?:\*\*|__)?\(?\d{1,3}[.)][\t ]+\S.*$/gm;
|
||||
|
||||
export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification {
|
||||
const plain = plainReview(input.text);
|
||||
@@ -263,7 +267,10 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
|
||||
const levelsOf = (c: string, p: string) => [...[...p.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity),
|
||||
...severityTokens(c), ...(reviewed ? severityWords(c) : [])];
|
||||
const levels = levelsOf(read, plainRead);
|
||||
const findings = { highest: levels.length ? levels.sort()[0]! : null };
|
||||
const verdicts = [lastVerdict(plainCode), lastVerdict(plainRead)];
|
||||
const unlabeled = reviewed && verdicts[1] !== undefined
|
||||
? [...read.matchAll(NUMBERED_ITEM)].filter(m => !levelsOf(m[0], plainReview(m[0])).length).length : 0;
|
||||
const findings = { highest: levels.length ? levels.sort()[0]! : null, ...(unlabeled ? { unlabeled } : {}) };
|
||||
const cleanEvidence = levelsOf(code, plainCode).length > 0;
|
||||
const ran = execution(input);
|
||||
if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail };
|
||||
@@ -283,9 +290,11 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
|
||||
if (!reviewed) return result(blocking ? 'findings' : 'clean');
|
||||
const unparsed = firstUnparsed(read);
|
||||
// A last explicit not-ready blocks even when the only parsed severities are advisory.
|
||||
const notReady = verdictNotReady(plainCode) || verdictNotReady(plainRead);
|
||||
const notReady = verdicts.some(v => v !== undefined && /^not[\t ]+ready\b/i.test(v));
|
||||
if (blocking || notReady) return { ...result('findings'), ...(unparsed ? { unparsed } : {}) };
|
||||
if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed };
|
||||
const unready = verdicts.find(v => v !== undefined && (!/^ready\b/i.test(v) || UNVERIFIED_CLAIM.test(v)));
|
||||
if (unready !== undefined) return result('unverified', 'unverified_verdict', `Verdict: ${unready}`.trim().slice(0, 160));
|
||||
if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(code)) {
|
||||
return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
|
||||
}
|
||||
@@ -388,7 +397,7 @@ if (import.meta.main) {
|
||||
const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events });
|
||||
const label = flags['--label'] ?? 'Outside review';
|
||||
console.log(`VERDICT: ${checked.verdict}`);
|
||||
console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`);
|
||||
console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' || checked.findings.unlabeled ? 'unspecified' : 'none')}`);
|
||||
if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`);
|
||||
if (flags['--events'] !== undefined) console.log(formatUsageLine(usageFromEvents(events)));
|
||||
if (checked.reason) {
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# Codex reply shapes from #3925
|
||||
|
||||
Issue #3925 reports two Codex consult replies that the outside-review helper
|
||||
summarised as `VERDICT: clean / FINDINGS: none`. The issue describes them but
|
||||
does not include them, so these files are synthesized from what it quotes:
|
||||
|
||||
- `cant-verify.txt`: a reply whose last line is `Verdict: Can't verify (...)`.
|
||||
The reviewer says it could not finish, so it is not `clean`.
|
||||
- `nonblocking-numbered.txt`: one numbered finding labeled `Nonblocking`, no
|
||||
P0-P3 tag, then `Verdict: ready` and `END OF REPORT: 1 findings (0 blocking)`.
|
||||
The verdict is `clean`, but `FINDINGS:` must not say `none`.
|
||||
- `ready-clean.txt`: the negative control, a real clean reply with
|
||||
`Verdict: ready` and no findings.
|
||||
|
||||
`test/outside-review-unparsed-severity.test.ts` holds the expected verdicts.
|
||||
@@ -0,0 +1,5 @@
|
||||
I read the diff for the files I could open. `lib/sync.ts` was excluded from
|
||||
the review scope, and the four required suites could not start in the sandbox,
|
||||
so I could not run them.
|
||||
|
||||
Verdict: Can't verify (one changed file excluded and four required suites blocked)
|
||||
@@ -0,0 +1,6 @@
|
||||
I read the diff and ran `bun test test/retry.test.ts`.
|
||||
|
||||
1. **Nonblocking — The retry log line repeats the attempt count.** `lib/retry.ts:41`
|
||||
|
||||
Verdict: ready
|
||||
END OF REPORT: 1 findings (0 blocking)
|
||||
@@ -0,0 +1,5 @@
|
||||
I read the diff, ran `bun test test/retry.test.ts`, and checked every caller of
|
||||
`retry()`. The cap is enforced and the tests cover the boundary.
|
||||
|
||||
Verdict: ready
|
||||
END OF REPORT: 0 findings (0 blocking)
|
||||
@@ -176,7 +176,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-margin-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-packet-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts'],
|
||||
'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts', 'test/outside-review-unparsed-severity.test.ts'],
|
||||
'test/fixtures/outside-voice-failure-prose-allowlist.json': ['test/outside-voice-failure-policy.test.ts'],
|
||||
'test/fixtures/overlay-v4-replay-37198445662.json': ['test/overlay-measurement.test.ts'],
|
||||
'test/fixtures/paid-report-census/**': ['test/paid-report-census.test.ts', 'test/paid-shards.test.ts'],
|
||||
|
||||
@@ -7,6 +7,9 @@
|
||||
* forms the issue documents (it describes 13 replies but does not include them).
|
||||
* #3329: that same gate then printed `clean` for a parsed advisory finding
|
||||
* whose last `Verdict:` line says `not ready`. `bare-dash-p2.txt` is unchanged.
|
||||
* #3925: it printed `clean` for a `Verdict: Can't verify` reply, and
|
||||
* `FINDINGS: none` for a numbered finding labeled `Nonblocking` with no P-tag
|
||||
* (fixtures in test/fixtures/outside-review-severity/codex-3925/).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -184,3 +187,62 @@ describe('#3329 a last not-ready Verdict blocks even with advisory findings', ()
|
||||
expect([failing.verdict, failing.score]).toEqual(['findings', 6]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#3925 an unrecognized verdict is not clean; numbered findings never print FINDINGS: none', () => {
|
||||
const FIX3925 = path.join(ROOT, 'test', 'fixtures', 'outside-review-severity', 'codex-3925');
|
||||
const cli3925 = (gate: OutsideGate, name: string) =>
|
||||
spawnSync(process.execPath, [LIB, '--verdict', '--label', 'Codex consult', '--exit', '0', gate, path.join(FIX3925, name)], { encoding: 'utf8', timeout: 10000 });
|
||||
const textFor = (gate: OutsideGate, text: string) => gate === 'review' ? `${text}\n${REC}` : text;
|
||||
const see = (text: string, gate: OutsideGate) => {
|
||||
const r = classifyOutsideReview({ text: textFor(gate, text), gate });
|
||||
return { verdict: r.verdict as string, reason: (r.reason ?? null) as string | null, unlabeled: r.findings.unlabeled ?? 0 };
|
||||
};
|
||||
|
||||
for (const [name, status, stdout] of [
|
||||
['cant-verify.txt', 4, 'VERDICT: unverified\nFINDINGS: none\nREASON: unverified_verdict\n'],
|
||||
['nonblocking-numbered.txt', 0, 'VERDICT: clean\nFINDINGS: unspecified\n'],
|
||||
['ready-clean.txt', 0, 'VERDICT: clean\nFINDINGS: none\n'],
|
||||
] as const) {
|
||||
test(`${name}: exit ${status}`, () => {
|
||||
const r = cli3925('execution', name);
|
||||
expect([r.status, r.stdout]).toEqual([status, stdout]);
|
||||
});
|
||||
}
|
||||
|
||||
test('a cannot-verify verdict prints its reason, the quoted verdict and the anchor', () => {
|
||||
const r = cli3925('execution', 'cant-verify.txt');
|
||||
expect(r.stderr).toContain('Codex consult: ran, verdict unverified (');
|
||||
expect(r.stderr).toContain("Verdict: Can't verify (one changed file excluded");
|
||||
expect(r.stderr).toContain('docs/troubleshooting.md#outside-review-unverified-verdict');
|
||||
expect(GATE_OUTCOMES.unverified_verdict).toMatchObject({ state: 'ran', anchor: 'outside-review-unverified-verdict' });
|
||||
expect(validateOutsideReview(read('no-findings.txt').replace('Verdict: ready', "Verdict: Can't verify"), 'execution').completed).toBe(false);
|
||||
});
|
||||
|
||||
test('every verdict other than ready, and a ready verdict that says it could not verify, is unverified on every review gate', () => {
|
||||
for (const gate of ['execution', 'structured', 'review'] as const)
|
||||
for (const verdictLine of [
|
||||
"Verdict: Can't verify (suites blocked)", 'Verdict: cannot verify', 'Verdict: Unable to verify the Windows path',
|
||||
"**Verdict:** Can\u2019t verify", '### Verdict: blocked', 'Verdict: revise', 'Verdict:',
|
||||
'Verdict: ready, but I could not verify the migration', 'Verdict: ready (unverified)',
|
||||
])
|
||||
expect({ gate, verdictLine, ...see(`No issues found.\n${verdictLine}`, gate) }).toEqual({ gate, verdictLine, verdict: 'unverified', reason: 'unverified_verdict', unlabeled: 0 });
|
||||
});
|
||||
|
||||
test('numbered findings without a P-tag are counted, whatever their label', () => {
|
||||
for (const label of ['**Nonblocking — The log repeats.**', 'Nonblocking: the log repeats.', '**Minor** — the log repeats.', 'The log repeats the attempt count.'])
|
||||
expect({ label, ...see(`1. ${label}\nVerdict: ready`, 'execution') }).toEqual({ label, verdict: 'clean', reason: null, unlabeled: 1 });
|
||||
expect(see('1. **P3 — Rename it.**\n2. Nonblocking — the log repeats.\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 1 });
|
||||
expect(classifyOutsideReview({ text: fs.readFileSync(path.join(FIX3925, 'nonblocking-numbered.txt'), 'utf8'), gate: 'structured' })).toMatchObject({ verdict: 'unverified', reason: 'untagged_review', findings: { highest: null, unlabeled: 1 } });
|
||||
});
|
||||
|
||||
test('negative controls: real clean replies, consult answers, tagged findings and quoted examples stay as they were', () => {
|
||||
for (const gate of ['execution', 'structured', 'review'] as const) {
|
||||
expect({ gate, ...see(read('no-findings.txt'), gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 });
|
||||
expect({ gate, ...see('No issues found.\nVerdict: Ready to merge.', gate) }).toEqual({ gate, verdict: 'clean', reason: null, unlabeled: 0 });
|
||||
}
|
||||
expect(see('1. Use a queue.\n2. Add a retry cap.', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
|
||||
expect(see('1. **P2 — The docs lag.**\n2. **P3 — Rename it.**\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
|
||||
expect(see('No issues found.\n```\n1. Nonblocking — example\nVerdict: cannot verify\n```\nVerdict: ready', 'execution')).toEqual({ verdict: 'clean', reason: null, unlabeled: 0 });
|
||||
expect(see('1. The retry path can double-send.\nVerdict: not ready', 'execution')).toEqual({ verdict: 'findings', reason: null, unlabeled: 1 });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user