/** * Review-specific completion evidence, separate from provider transport success. * * classifyOutsideReview() is the INV-1 contract: it takes the review text, the * provider's stderr and exit status (and, for `codex exec --json`, the event * stream) and returns three separate answers: did the review execute, what is * the highest finding severity, and the verdict. validateOutsideReview() is the * older text-only shape, kept for direct importers and stale rendered skills. */ import { GATE_OUTCOMES, gateOutcomeLine, type GateReason } from './gate-outcomes'; export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution' | 'proposal'; export type OutsideVerdict = 'clean' | 'findings' | 'unverified' | 'unavailable'; export type Severity = 'P0' | 'P1' | 'P2' | 'P3'; export interface OutsideReviewInput { text: string; gate: OutsideGate; stderr?: string; exit?: number; /** JSONL from `codex exec --json`: command events are positive execution evidence. */ events?: string; } export interface OutsideReviewClassification { execution: { state: 'ran' } | { state: 'unavailable'; reason: GateReason; detail?: string }; findings: { highest: Severity | null }; verdict: OutsideVerdict; /** Set for every verdict except clean/findings. */ reason?: GateReason; detail?: string; score?: number; /** A P0-P3 label the parser could not read, quoted; set beside parsed blocking findings too. */ unparsed?: string; } const REFUSAL = /\b(?:(?:I (?:cannot|can't|won't|will not|am unable to)|I'm unable to)\s+(?:review|analy[sz]e|evaluate|assess|inspect|access|complete|perform|provide|assist|help|proceed)|unable to (?:review|analy[sz]e)|I must (?:decline|refuse))\b/i; /** * Sandbox setup failures as Codex and bubblewrap print them on stderr (fixtures: * test/fixtures/codex-sandbox/). Read from stderr and failed command output * only: a review that merely discusses bwrap or namespaces must still pass, and * a healthy run warns "could not find bubblewrap on PATH" while working. */ const SANDBOX_FAILURE = /^\s*bwrap: \S.*$|^.*\bbubblewrap is unavailable\b.*$|^.*\blandlock\b.{0,60}\b(?:fail\w*|error|not supported|unsupported)\b.*$|^.*\bseccomp\b.{0,60}\b(?:fail\w*|error)\b.*$|^.*\buser namespaces?\b.{0,80}\b(?:not (?:allowed|permitted|supported)|denied|disabled)\b.*$/im; /** Exact phrases a reviewer uses when it could not execute; the last fallback, used only without positive evidence. */ const EXECUTION_FAILURE_PHRASE = /\b(?:commands? (?:could not|couldn't|cannot|can't) (?:be )?run|(?:could not|couldn't|was unable to|am unable to|unable to) (?:run (?:any )?(?:shell )?commands|execute (?:any )?commands|inspect the diff|read the diff|access the diff)|the diff could not be (?:read|inspected|accessed)|every (?:shell )?(?:command|invocation) failed)\b/i; /** * Codex's own trailing error lines (`ERROR: ...`, `stream error: ...`), never * the whole stderr: it echoes the session prompt, which may quote any of the * phrases below. Mirrors the awk filter in bin/gstack-codex-probe. */ const CODEX_ERROR_LINE = /^\s*(?:\[[^\]]*\]\s*)?(?:ERROR:|stream error)/; function codexErrorLines(stderr: string): string[] { let block: string[] = []; for (const line of stderr.split(/\r?\n/)) { if (CODEX_ERROR_LINE.test(line)) block.push(line.trim()); else if (line.trim()) block = []; } return block; } /** The account's usage limit: Codex's line carries the reset time, so it is relayed verbatim. Same signature as the probe. */ export const QUOTA_FAILURE = /usage limit|insufficient_quota|exceeded your current quota|quota exceeded/i; /** A plain HTTP 429 or rate limit: transient, so a separate reason from the quota. */ export const RATE_LIMIT_FAILURE = /rate.?limit|too many requests|(?:^|[^0-9])429(?:[^0-9]|$)/i; /** `codex review` transcript on stderr: a command that ran prints " succeeded in Nms:". */ const TRANSCRIPT_SUCCESS = /^\s*succeeded in \d+(?:\.\d+)?m?s:?\s*$/m; function commandEvidence(events: string): { attempted: number; succeeded: number; failedOutput: string } { let attempted = 0, succeeded = 0, failedOutput = ''; for (const line of events.split(/\r?\n/)) { if (!line.trim().startsWith('{')) continue; let event: any; try { event = JSON.parse(line); } catch { continue; } const item = event?.item; if (event?.type !== 'item.completed' || item?.type !== 'command_execution') continue; attempted++; if (item.status === 'completed' && item.exit_code === 0) succeeded++; else failedOutput += `${typeof item.aggregated_output === 'string' ? item.aggregated_output : ''}\n`; } return { attempted, succeeded, failedOutput }; } function stderrHead(stderr: string): string | undefined { return stderr.split(/\r?\n/).map(line => line.trim()).find(Boolean)?.slice(0, 240); } function execution(input: OutsideReviewInput): OutsideReviewClassification['execution'] { const stderr = input.stderr ?? ''; const exit = input.exit ?? 0; const sandbox = (output: string) => output.match(SANDBOX_FAILURE)?.[0].trim().slice(0, 240); if (exit !== 0) { const detail = sandbox(stderr); if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail }; const errors = exit !== 124 ? codexErrorLines(stderr) : []; const quota = errors.find(line => QUOTA_FAILURE.test(line)); if (quota) return { state: 'unavailable', reason: 'quota_exhausted', detail: quota.slice(0, 240) }; const rateLimit = errors.find(line => RATE_LIMIT_FAILURE.test(line)); if (rateLimit) return { state: 'unavailable', reason: 'rate_limited', detail: rateLimit.slice(0, 240) }; const head = stderrHead(stderr); return { state: 'unavailable', reason: exit === 124 ? 'timeout' : 'execution_failed', detail: head ? `exit ${exit}: ${head}` : `exit ${exit}` }; } const commands = commandEvidence(input.events ?? ''); const executed = commands.succeeded > 0 || TRANSCRIPT_SUCCESS.test(stderr); if (!executed) { const detail = sandbox(stderr) ?? sandbox(commands.failedOutput); if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail }; if (commands.attempted > 0) return { state: 'unavailable', reason: 'commands_failed', detail: `all ${commands.attempted} commands failed` }; } if (!input.text.trim()) return { state: 'unavailable', reason: 'empty_response' }; const phrase = executed ? undefined : input.text.match(EXECUTION_FAILURE_PHRASE)?.[0]; if (phrase) return { state: 'unavailable', reason: 'commands_failed', detail: `the review says "${phrase}"` }; if (REFUSAL.test(input.text)) return { state: 'unavailable', reason: 'review_refused' }; return { state: 'ran' }; } /** * Severity words count as findings only in label position, never inside prose * ("high-level", "a low-risk change", "no critical findings"): `Severity: High` * or `Priority: high`, a leading `High:` / `High —` / `[High]` (after an * optional heading, bullet or number), a bold `**High**`, a table cell, or a * trailing `… — Medium.` / `… (High)` at the end of a finding line. * Critical and high block like P0/P1; medium and low are advisory like P2/P3. */ const SEVERITY_WORDS: Record = { critical: 'P0', high: 'P1', medium: 'P2', low: 'P3' }; const WORD = '(critical|high|medium|low)'; const SEVERITY_LABELS = [ new RegExp(`\\b(?:severity|priority)\\b[\\t ]*[:=][\\t ]*(?:\\*\\*|__|\\[|\`)*${WORD}\\b`, 'gim'), new RegExp(`^[\\t ]*(?:>[\\t ]*)?(?:#{1,6}[\\t ]+|[-+*][\\t ]+|\\(?\\d{1,3}[.)][\\t ]+)?(?:\\*\\*|__)?\\[?${WORD}\\]?(?:\\*\\*|__)?[\\t ]*(?::|\u2014|\u2013|-[\\t ]|\\]|\\(|\\*\\*[\\t ]*(?:\u2014|\u2013|-[\\t ]))`, 'gim'), new RegExp(`(?:\\*\\*|__)\\[?${WORD}\\]?:?(?:\\*\\*|__)`, 'gi'), new RegExp(`\\|[\\t ]*(?:\\*\\*)?${WORD}(?:\\*\\*)?[\\t ]*(?=\\|)`, 'gi'), new RegExp(`[\\t ](?:\u2014|\u2013|-)[\\t ]+(?:\\*\\*|__)?${WORD}(?:\\*\\*|__)?[\\t ]*[.;]?[\\t ]*$`, 'gim'), new RegExp(`\\((?:severity:[\\t ]*)?${WORD}\\)[\\t ]*[.;]?[\\t ]*$`, 'gim'), ]; function severityWords(text: string): Severity[] { return SEVERITY_LABELS.flatMap(re => [...text.matchAll(re)].map(m => SEVERITY_WORDS[m[1]!.toLowerCase()]!)); } /** * A no-findings statement opening a sentence: NO_FINDINGS (its underscore is * gone after plainReview), "No issues found", "I found no actionable * regressions", "I did not find any bugs". Never a phrase inside another * clause ("renaming the flag is no problem for callers, but ..."). */ const FINDING_NOUN = '(?:(?:actionable|significant|new|concrete)\\s+)?(?:bugs?|issues?|findings?|problems?|regressions?|defects?)\\b'; const NO_FINDINGS_LEAD = '(?:(?:overall|in summary|in short|summary|conclusion|result)[,:]?\\s+|(?:after|having|on|upon)\\s+(?:reviewing|reading|checking|inspecting|reviewed|read|checked|inspected)\\b[^,.;]{0,80},\\s+)?'; const NO_FINDINGS = new RegExp(`^${NO_FINDINGS_LEAD}(?:NO_?FINDINGS\\b|(?:(?:i|we)\\s+(?:found|see|saw|identified)\\s+|there\\s+(?:are|were|is)\\s+)?no\\s+${FINDING_NOUN}|(?:i|we)\\s+(?:did not|didn't|could not|couldn't|cannot|can't)\\s+(?:find|identify|see)\\s+any\\s+${FINDING_NOUN})`, 'i'); /** A contrast or exception qualifies the claim, so the sentence does not conclude that nothing was found. */ const NO_FINDINGS_EXCEPTION = /(? sentence.trim()).filter(Boolean); const total = sentences.length + (because ? 1 : 0); return [sentences.at(-1), because].some(conclusion => { const statement = conclusion?.match(NO_FINDINGS)?.[0]; if (!conclusion || !statement || NO_FINDINGS_EXCEPTION.test(conclusion)) return false; const qualifier = conclusion.slice(statement.length).replace(NO_FINDINGS_PREDICATE, '').replace(/[\s.!]+$/, '').trim(); return total === 1 || !qualifier || qualifier.startsWith(':') || WHOLE_CHANGE_SCOPE.test(qualifier); }); } /** Formatting the requested marker in bold, inline code, or a list does not invalidate a completed review. */ function plainReview(text: string): string { return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n'); } /** * Code quotes text, so a severity inside a fence or an inline code span is not * a label. An inline span holding only a severity token in label position * (`- \`P1\`: x`, `Severity: \`High\``, a table cell) is formatting and stays. */ const CODE_LABEL_BEFORE = /^[\t ]*(?:>[\t ]*)?(?:#{1,6}[\t ]+|[-+*][\t ]+|\(?\d{1,3}[.)][\t ]+)?(?:\*\*|__)?$|\b(?:severity|priority)[\t ]*[:=][\t ]*(?:\*\*|__)?$|\|[\t ]*(?:\*\*)?$/i; function withoutCode(text: string): string { let fence: string | undefined; return text.split(/\r?\n/).map(line => { const marker = /^[\t ]*(`{3,}|~{3,})/.exec(line)?.[1]; if (fence) { if (marker && marker[0] === fence[0] && marker.length >= fence.length) fence = undefined; return ''; } if (marker) { fence = marker; return ''; } return line.replace(/`([^`]+)`/g, (span, inner: string, at: number) => inner.trim() === 'NO_FINDINGS' || /^\[?(?:P[0-3]|critical|high|medium|low)\]?$/i.test(inner.trim()) && CODE_LABEL_BEFORE.test(line.slice(0, at)) ? inner : ' '); }).join('\n'); } /** * P0-P3 in label position: at a line start after an optional quote, heading, * bullet, number or bold, as `[P1]`, `P1:`, `P1 —`, `**P1 — x**`, `**P1** — x`, * `P1 (x)` or a bare `P1` ending the line; or a table cell `| P1 |`. */ const LABEL_START = '^[\\t ]*(?:>[\\t ]*)?(?:#{1,6}[\\t ]+|[-+*][\\t ]+|\\(?\\d{1,3}[.)][\\t ]+)?(?:\\*\\*|__)?'; const LABELED_P = new RegExp(`${LABEL_START}(?:\\[(P[0-3])\\]|(P[0-3])(?:\\*\\*|__)?[\\t ]*(?::|\u2014|\u2013|-[\\t ]|\\(|$|(?:\\*\\*|__)[\\t ]*(?::|\u2014|\u2013|-[\\t ]|$)))`, 'gm'); const TABLE_P = /\|[\t ]*(?:\*\*)?\[?(P[0-3])\]?(?:\*\*)?[\t ]*(?=\|)/g; const LABEL_P = new RegExp(`${LABEL_START}\\[?P[0-3]\\b`); function severityTokens(text: string): Severity[] { return [...[...text.matchAll(LABELED_P)].map(m => (m[1] ?? m[2]) as Severity), ...[...text.matchAll(TABLE_P)].map(m => m[1] as Severity)]; } /** The first line with a P0-P3 token in label position that no severity form reads. */ function firstUnparsed(text: string): string | undefined { const lines = text.split('\n'); const at = lines.findIndex(line => LABEL_P.test(line) && !new RegExp(LABELED_P.source, 'm').test(line)); return at < 0 ? undefined : `line ${at + 1}: "${lines[at]!.trim().slice(0, 160)}"`; } /** The reply's last `Verdict:` line says it is not ready. */ function verdictNotReady(plain: string): boolean { const last = [...plain.matchAll(/^[\t ]*Verdict[\t ]*:[\t ]*([^\r\n]*)$/gim)].at(-1)?.[1]; return last !== undefined && /^not[\t ]+ready\b/i.test(last.trim()); } export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification { const plain = plainReview(input.text); const reviewed = input.gate !== 'proposal'; const code = withoutCode(input.text), plainCode = plainReview(code); const levels = [...[...plainCode.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity), ...severityTokens(code), ...(reviewed ? severityWords(code) : [])]; const findings = { highest: levels.length ? levels.sort()[0]! : null }; const ran = execution(input); if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail }; const blocking = findings.highest === 'P0' || findings.highest === 'P1'; const result = (verdict: OutsideVerdict, reason?: GateReason, detail?: string): OutsideReviewClassification => ({ execution: ran, findings, verdict, ...(reason ? { reason } : {}), ...(detail ? { detail } : {}) }); if (input.gate === 'spec') { const scores = [...input.text.matchAll(/^SCORE:[\t ]*(10|[0-9])[\t ]*\r?$/gm)]; const ambiguities = [...input.text.matchAll(/^AMBIGUITIES:[\t ]*(\S[^\r\n]*)\r?$/gm)]; if (scores.length !== 1 || ambiguities.length !== 1) return result('unavailable', 'missing_markers', 'missing or invalid SCORE/AMBIGUITIES markers'); const score = Number(scores[0]![1]); return { ...result(score >= 7 ? 'clean' : 'findings'), score }; } if ((input.gate === 'review' || input.gate === 'proposal') && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) { return result('unavailable', 'missing_markers', 'missing review completion recommendation'); } if (!reviewed) return result(blocking ? 'findings' : 'clean'); const unparsed = firstUnparsed(code); // A last explicit not-ready blocks even when the only parsed severities are advisory. const notReady = verdictNotReady(plainCode); if (blocking || notReady) return { ...result('findings'), ...(unparsed ? { unparsed } : {}) }; if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed }; if ((input.gate === 'structured' || input.gate === 'review') && !findings.highest && !concludesNoFindings(code)) { return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion'); } return result('clean'); } const LEGACY_REASONS: Partial> = { empty_response: 'empty response', review_refused: 'review refused', }; /** Text-only compatibility shape for direct importers (claude-code/SKILL.md.tmpl) and the two-argument CLI. */ export function validateOutsideReview(text: string, gate: OutsideGate): { completed: boolean; reason?: string; score?: number; gate?: 'pass' | 'fail' } { const checked = classifyOutsideReview({ text, gate }); if (checked.verdict === 'unavailable' || checked.verdict === 'unverified') { const reason = LEGACY_REASONS[checked.reason!] ?? checked.detail ?? GATE_OUTCOMES[checked.reason!].summary; return { completed: false, reason }; } if (gate === 'spec') return { completed: true, score: checked.score, gate: checked.verdict === 'clean' ? 'pass' : 'fail' }; if (gate === 'structured') return { completed: true, gate: checked.verdict === 'findings' ? 'fail' : 'pass' }; return { completed: true }; } const VERDICT_EXIT: Record = { clean: 0, findings: 3, unverified: 4, unavailable: 1 }; const GATES = ['review', 'structured', 'spec', 'execution', 'proposal']; const USAGE = `Usage: outside-review-result.ts outside-review-result.ts --verdict [--stderr ] [--exit ] [--events ] [--label ] Gates: ${GATES.join('|')}. Two-argument form exits 0 completed, 1 unavailable, 2 usage. Verdict form prints VERDICT: clean|findings|unverified|unavailable and exits 0, 3, 4 or 1.`; async function readOptional(file: string | undefined): Promise { if (!file) return ''; try { return await Bun.file(file).text(); } catch { return ''; } } if (import.meta.main) { const args = process.argv.slice(2); const flags: Record = {}; const positional: string[] = []; let verdictMode = false; for (let i = 0; i < args.length; i++) { const arg = args[i]!; if (arg === '--verdict') verdictMode = true; else if (['--stderr', '--exit', '--events', '--label'].includes(arg) && i + 1 < args.length) { flags[arg] = args[++i]!; verdictMode ||= arg !== '--label'; } else if (arg.startsWith('--')) { console.error(USAGE); process.exit(2); } else positional.push(arg); } const [gate, path] = positional; const exit = flags['--exit'] === undefined ? 0 : Number(flags['--exit']); if (!gate || !GATES.includes(gate) || !path || positional.length !== 2 || !Number.isInteger(exit) || (!verdictMode && gate === 'execution')) { console.error(USAGE); process.exit(2); } let text: string; try { text = await Bun.file(path).text(); } catch (error) { if (!verdictMode) { console.error(`Outside review unavailable: ${error}`); process.exit(1); } text = ''; } if (!verdictMode) { const result = validateOutsideReview(text, gate as OutsideGate); if (!result.completed) { console.error(`Outside review unavailable: ${result.reason}; missing coverage.`); process.exit(1); } process.exit(0); } const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events: await readOptional(flags['--events']) }); const label = flags['--label'] ?? 'Outside review'; console.log(`VERDICT: ${checked.verdict}`); console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`); if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`); if (checked.reason) { console.log(`REASON: ${checked.reason}`); console.error(gateOutcomeLine(label, checked.reason, checked.detail)); } else if (checked.unparsed) { const outcome = GATE_OUTCOMES.unparsed_severity; console.error(`${label}: findings block this review; separately, ${outcome.summary} (${checked.unparsed}). Fix: ${outcome.fix}.`); } process.exit(VERDICT_EXIT[checked.verdict]); }