fix(outside-review): severity words in label position count as findings

The Oct 7 /autoplan run's Codex CEO and DX reviews labeled seven and eight
findings High/Medium without [Pn] tags, and the classifier printed
VERDICT: clean / FINDINGS: none for both. lib/outside-review-result.ts now
reads severity words in label position (Severity: High, a leading High: /
High — / [High], **High**, a table cell): critical/high block like P0/P1,
medium/low are advisory. Words in prose (high-level, low-risk, no critical
findings) never count. A review or structured response with neither tags,
labels nor an explicit no-findings statement is unverified, never clean
(DX-11, ENG-10). Design-direction proposals move to a new proposal gate so
they keep needing only their Recommendation line, and review prompts ask
for a severity per finding. Fixtures reproduce the CEO/DX label shapes, the
eng [P1] output, a table and prose negative controls; the stored Codex panel
keeps its verdicts. docs/troubleshooting.md explains how the verdict is read.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-07 20:14:21 +00:00
1 parent 64e8f90f15
commit 8aae497d2b
33 files changed
+276 -45

No files matched your search

+29 -9
View File
@@ -9,7 +9,7 @@
*/
import { GATE_OUTCOMES, gateOutcomeLine, type GateReason } from './gate-outcomes';
export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution';
export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution' | 'proposal';
export type OutsideVerdict = 'clean' | 'findings' | 'unverified' | 'unavailable';
export type Severity = 'P0' | 'P1' | 'P2' | 'P3';
@@ -112,6 +112,27 @@ function execution(input: OutsideReviewInput): OutsideReviewClassification['exec
return { state: 'ran' };
}
/**
* Severity words count as findings only in label position, never inside prose
* ("high-level", "a low-risk change", "no critical findings"): `Severity: High`
* or `Priority: high`, a leading `High:` / `High —` / `[High]` (after an
* optional heading, bullet or number), a bold `**High**`, or a table cell.
* Critical and high block like P0/P1; medium and low are advisory like P2/P3.
*/
const SEVERITY_WORDS: Record<string, Severity> = { critical: 'P0', high: 'P1', medium: 'P2', low: 'P3' };
const WORD = '(critical|high|medium|low)';
const SEVERITY_LABELS = [
new RegExp(`\\b(?:severity|priority)\\b[\\t ]*[:=][\\t ]*(?:\\*\\*|__|\\[|\`)*${WORD}\\b`, 'gim'),
new RegExp(`^[\\t ]*(?:>[\\t ]*)?(?:#{1,6}[\\t ]+|[-+*][\\t ]+|\\(?\\d{1,3}[.)][\\t ]+)?(?:\\*\\*|__)?\\[?${WORD}\\]?(?:\\*\\*|__)?[\\t ]*(?::|\u2014|\u2013|-[\\t ]|\\]|\\(|\\*\\*[\\t ]*(?:\u2014|\u2013|-[\\t ]))`, 'gim'),
new RegExp(`(?:\\*\\*|__)\\[?${WORD}\\]?:?(?:\\*\\*|__)`, 'gi'),
new RegExp(`\\|[\\t ]*(?:\\*\\*)?${WORD}(?:\\*\\*)?[\\t ]*(?=\\|)`, 'gi'),
];
function severityWords(text: string): Severity[] {
return SEVERITY_LABELS.flatMap(re => [...text.matchAll(re)].map(m => SEVERITY_WORDS[m[1]!.toLowerCase()]!));
}
/** An explicit conclusion that the review found nothing to report. */
const NO_FINDINGS = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs?|issues?|findings?|problems?)\b|\b(?:did not|didn't|cannot|can't|could not|couldn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i;
/** Formatting the requested marker in bold, inline code, or a list does not invalidate a completed review. */
function plainReview(text: string): string {
return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n');
@@ -119,7 +140,8 @@ function plainReview(text: string): string {
export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification {
const plain = plainReview(input.text);
const levels = [...plain.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity);
const levels = [...[...plain.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity),
...(input.gate === 'review' || input.gate === 'structured' ? severityWords(input.text) : [])];
const findings = { highest: levels.length ? levels.sort()[0]! : null };
const ran = execution(input);
if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail };
@@ -133,14 +155,12 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
const score = Number(scores[0]![1]);
return { ...result(score >= 7 ? 'clean' : 'findings'), score };
}
if (input.gate === 'structured') {
const clear = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs|issues|findings|problems)\b|\b(?:did not|didn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i.test(input.text);
if (!findings.highest && !clear) return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
return result(blocking ? 'findings' : 'clean');
}
if (input.gate === 'review' && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) {
if ((input.gate === 'review' || input.gate === 'proposal') && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) {
return result('unavailable', 'missing_markers', 'missing review completion recommendation');
}
if ((input.gate === 'structured' || input.gate === 'review') && !findings.highest && !NO_FINDINGS.test(input.text)) {
return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
}
return result(blocking ? 'findings' : 'clean');
}
@@ -162,7 +182,7 @@ export function validateOutsideReview(text: string, gate: OutsideGate): { comple
}
const VERDICT_EXIT: Record<OutsideVerdict, number> = { clean: 0, findings: 3, unverified: 4, unavailable: 1 };
const GATES = ['review', 'structured', 'spec', 'execution'];
const GATES = ['review', 'structured', 'spec', 'execution', 'proposal'];
const USAGE = `Usage: outside-review-result.ts <gate> <response-file>
outside-review-result.ts --verdict [--stderr <file>] [--exit <code>] [--events <file>] [--label <name>] <gate> <response-file>
Gates: ${GATES.join('|')}. Two-argument form exits 0 completed, 1 unavailable, 2 usage.