diff --git a/bin/gstack-review-dispatch b/bin/gstack-review-dispatch index 6172bff23..676056598 100755 --- a/bin/gstack-review-dispatch +++ b/bin/gstack-review-dispatch @@ -9,7 +9,7 @@ * gstack-review-dispatch judge --ledger --merged --judged no drop, no downgrade * gstack-review-dispatch verify --ledger [--targeted-out ] replay every hash and the judgment * gstack-review-dispatch pass --base --out --pass [-- plan flags] the pass's staged manifest - * gstack-review-dispatch show|stage|chunk|mark --manifest ... one manifest step + * gstack-review-dispatch show|stage|chunk|mark|retry --manifest ... one manifest step * gstack-review-dispatch program [--ledger ] the checked-in Capy pass program * gstack-review-dispatch audit-fetch --events [--manifest --reader ] what a reader fetched (GST-1322) * gstack-review-dispatch sweep --manifest --stage --phase start|end what a stage fetched; quarantine and rerun @@ -24,7 +24,9 @@ import * as path from 'node:path'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { BRIEF_ROLES, BriefError, briefFile, checkReviewedTree, checkoutRoot, checkoutTree, promptHash, renderBrief, sectionTexts, sliceDiff, sliceNote, withWorkspace, type BriefRole, type Slots, type TreeCheck } from '../lib/review-brief'; import { cleanMarkers, collect, fixableRaises, mayBeUnavailable, parseFindings, readJsonl, readLedger, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine, type VerifyRow } from '../lib/review-ledger'; -import { buildManifest, chunkCount, chunkOf, manifestPath, markReader, readManifest, renderStage, withAuditModel, writeManifest, STAGES, type Extra, type Manifest, type PassReader, type ReaderState, type Stage } from '../lib/review-pass'; +import { buildManifest, chunkCount, chunkOf, manifestPath, markReader, planCyberRetry, readManifest, renderStage, withAuditModel, writeManifest, STAGES, type Extra, type Manifest, type PassReader, type ReaderState, type Stage } from '../lib/review-pass'; +import { classifyOutsideReview } from '../lib/outside-review-result'; +import { gateOutcomeLine } from '../lib/gate-outcomes'; import { diffCommand, type Dispatch } from '../lib/review-depth'; import { mayExclude, policySelection, selectFor } from '../lib/review-selection'; import { REVIEW_PLAN_TIMEOUT_MS } from '../lib/review-collect'; @@ -102,6 +104,10 @@ const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free specialists' merged.jsonl lines byte for byte chunk --manifest --reader --index BRIEF_CHUNK: reader=… index=i of=n sha256=…, then one JSON string line of at most 65,536 brief characters + retry --manifest --reader --error-file + the host's error for an outside voice that ended without a review, classified by lib/outside-review-result.ts: a cyber_policy + refusal plans R95's next attempt (GPT-6 Astra on the read-only brief, then Gemini 4 Carbon on the original) as RETRY: reader=..., + or RETRY: exhausted once both ran; anything else prints RETRY: none and changes nothing mark --manifest --reader --state [--path

] [--output ] [--error ] an audit reader is collected with --output, whose REVIEWED_TREE line must match its render (else failed, COLLECT_WRONG_TREE) mark --manifest --before , [--voices] the pass's reading at its first dispatch; --voices keeps Step 11's own @@ -308,13 +314,13 @@ const branchDiff = (base: string, r?: Pick & { diff_base?: string }>, diffBase: string, known: PlanExclude | undefined): { exit: number; line: string } | null { +function ceiling(context: string, base: string, outDir: string, readers: ReadonlyArray & { diff_base?: string }>, diffBase: string, known: PlanExclude | undefined): { exit: number; line: string } | null { for (const r of readers) { if (r.prompt_file) continue; const at = r.diff_base ?? diffBase; const d = selectionOf(at, r.exclude, r.keep, r.excerpt); if (!d.ok) return { exit: EXIT.fail, line: unreadableLine(context, r.reader, d, at, base, outDir) }; - if (d.bytes > REVIEW_DIFF_MAX_BYTES && r.reader !== 'outside-voice') return { exit: EXIT.refused, line: tooLargeLine(context, r.reader, d, ceilingContext(process.cwd(), base, known)).line }; + if (d.bytes > REVIEW_DIFF_MAX_BYTES && r.role !== 'outside-voice') return { exit: EXIT.refused, line: tooLargeLine(context, r.reader, d, ceilingContext(process.cwd(), base, known)).line }; } return null; } @@ -457,6 +463,21 @@ function passStep(cmd: string): number { process.stdout.write([`STAGE: ${stage} rendered=${done.length}`, ...done.map(r => readerLine(m, r))].join('\n') + '\n'); return EXIT.ok; } + if (cmd === 'retry') { + const r = m.readers.find(x => x.reader === flag('--reader')); + if (r?.role !== 'outside-voice' || !flag('--error-file')) { process.stderr.write('gstack-review-dispatch retry: --manifest --reader --error-file \n'); return EXIT.usage; } + const checked = classifyOutsideReview({ text: '', gate: 'review', exit: 1, providerError: readFile(flag('--error-file'), 'host error') }); + if (checked.reason !== 'cyber_policy_refused') { process.stdout.write(`RETRY: none reader=${r.reader} (not a cyber_policy refusal; recorded as unavailable coverage as before)\n`); return EXIT.ok; } + let voices: { laneModel?: string | null; readers?: Record } = {}; + try { voices = JSON.parse(fs.readFileSync(path.join(m.out, 'voice-models.json'), 'utf8')); } catch { voices = {}; } + const taken = [voices.laneModel, voices.readers?.['adversarial-native']?.model_id].filter((id): id is string => typeof id === 'string' && id.includes('/')).map(id => id.split('/')[0]!.toLowerCase()); + const next = planCyberRetry(m, r, taken); + writeManifest(file, m); + process.stdout.write(`${gateOutcomeLine(`Outside voice ${r.reader}`, 'cyber_policy_refused', checked.detail)}\n${next + ? `RETRY: reader=${next.reader} model=${next.model!.model_id} brief=${next.read_only ? 'read-only' : 'original'} (R95: one attempt, never the same brief on another OpenAI model)` + : `RETRY: exhausted reader=${r.reader} (R95's attempts are spent: the second-family read is missing)`}\n`); + return EXIT.ok; + } if (cmd === 'chunk') { const r = m.readers.find(x => x.reader === flag('--reader')); if (!r?.file || !r.sha256) { process.stderr.write(`gstack-review-dispatch chunk: no rendered reader ${flag('--reader')}\n`); return EXIT.usage; } @@ -558,7 +579,7 @@ function main(): number { if (cmd === 'program') return program(); if (cmd === 'audit-fetch') return auditFetch(); if (cmd === 'sweep') return sweep(); - if (['show', 'stage', 'chunk', 'mark'].includes(cmd)) return passStep(cmd); + if (['show', 'stage', 'chunk', 'mark', 'retry'].includes(cmd)) return passStep(cmd); const ledger = flag('--ledger'); if (!ledger) { process.stderr.write(`gstack-review-dispatch ${cmd}: --ledger is required\n`); return EXIT.usage; } const reader = flag('--role') ? readerKey(flag('--role')!, flag('--name')) : ''; diff --git a/changelog.d/fix-outside-voice-cyber-policy-retry.md b/changelog.d/fix-outside-voice-cyber-policy-retry.md new file mode 100644 index 000000000..5322fe7df --- /dev/null +++ b/changelog.d/fix-outside-voice-cyber-policy-retry.md @@ -0,0 +1,4 @@ +--- +important: blocker +--- +- **When OpenAI refuses /ship's outside voice under its cyber-security policy, Capy's /ship now retries it instead of losing the second read.** A refusal that says `cyber_policy` is its own outcome now (`cyber_policy_refused`, docs/troubleshooting.md#outside-review-cyber-policy) instead of a generic failure. On Capy, the review pass sends the brief once more to GPT-6 Astra reworded as a read-only correctness review (no attacker role-play, no exploit code or reproductions, each defect as file:line, the wrong-answer condition and the smallest fix), and if Astra refuses too, once to Gemini 4 Carbon with the brief unchanged. If every attempt is refused, the Step 11 record says the second-family read is missing, the native review still runs, and the PR body lists the gap as missing coverage, not a pass. The first brief every outside voice gets is unchanged, other failures (auth, timeout, empty replies) are handled as before, and nothing else retries. The Codex CLI and api runners report the new reason but do not retry by themselves. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 315a3d55e..f6bf87f05 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -339,6 +339,33 @@ network. **Expected result.** `OUTSIDE_STATUS: completed`. + +### `... outside review unavailable: OpenAI declined the review under its cyber-security policy (cyber_policy) (...)` + +**Meaning.** OpenAI flagged the review brief as possible cybersecurity risk +and refused it (Capy relays it as `OpenAI key declined the request: +cyber_policy: ...`). It is not an auth, quota or model problem, and sending the +same brief to another OpenAI model gets the same answer: GPT-6.1 Sol and GPT-6 +Astra both refused the same two briefs in the 2026-11-13 test. + +**Fix.** Rerun the review once on GPT-6 Astra with the brief reworded as a +read-only correctness review (no attacker role-play, no exploit code or +reproductions; each defect as file:line, the condition under which the code +returns the wrong answer, and the smallest fix), or on Gemini 4 Carbon with the +brief unchanged. Capy's /ship does both by itself: the pass prints +`RETRY: reader=outside-voice/astra-read-only ...`, then +`RETRY: reader=outside-voice/gemini-4-carbon ...` if Astra refuses too. A +retry whose model family is already the lane's or the native reviewer's (an +OpenAI or Google lane, say) is skipped, because Step 11 refuses an outside +voice of either family. Other +hosts' outside voices (the Codex CLI, the api runner) only report this reason; +rerun them by hand. When every attempt is refused, `RETRY: exhausted` and the +Step 11 record's `outside_note` say the second-family read is missing: the +native review still runs and the PR body lists the gap as missing coverage, +never as a pass. + +**Expected result.** `OUTSIDE_RECORD: ... "outside_status":"completed","outside_note":"second-family read by ..."`. + ### `... outside review unavailable: the reviewer hit its output token limit and the response is cut off (...)` diff --git a/lib/capy-review-pass.js b/lib/capy-review-pass.js index aa6e1adad..38fed86e6 100644 --- a/lib/capy-review-pass.js +++ b/lib/capy-review-pass.js @@ -7,6 +7,8 @@ // reader, a brief mismatch or a failing bin returns `STOP:` for the parent; nothing is dropped. The outside // voice is the exception (GST-1104): one that hangs past stuckMs is stopped, and one that hangs, ends empty, or // stops or fails without findings is collected as unavailable coverage (one that left findings is collected); Step 11 returns OUTSIDE_RECORD: for its review-log rows. +// A cyber_policy refusal (R95) also plans the next attempt (`gstack-review-dispatch retry`): GPT-6 Astra on the read-only brief, +// then Gemini 4 Carbon on the original; the record names the last attempt and, once all were refused, the missing second-family read. // Once Phase 2 has started /parts (the parent of input.out), each poll or finished pass also // returns the next phase's section parts that fit its 32 KB (`gstack-ship-parts next`, GSTA-167 C5); // after a return that never reached the parent, rerun with input.resendParts to get that batch again. @@ -141,6 +143,35 @@ const recordUnavailable = async (m, r, why) => { await mark(r.reader, 'collected', { output: file, error: UNAVAILABLE + why }); return null; }; +const CYBER = 'refused under cyber_policy'; +const refusal = async (m, r, error) => { + const file = m.out + '/out/' + safe(r.reader) + '.host-error.txt'; + await tools.capy.write({ path: file, content: error }); + const c = await sh(bin + ' retry --manifest ' + q(mf) + ' --reader ' + q(r.reader) + ' --error-file ' + q(file)); + if (c.code !== 0) throw new Error('retry ' + r.reader + ' exited ' + c.code + ': ' + c.out.slice(-500)); + lines.push(...c.out.trim().split('\n')); + return c.out.includes('RETRY: none ') ? '' : CYBER + ' (' + error.trim().split('\n')[0].slice(0, 240) + ')'; +}; +const cyberWhy = (error) => { + const line = error.split(/\r?\n/).map((l) => l.trim()).find((l) => /\bcyber_policy\b/.test(l)); + return line ? CYBER + ' (' + line.slice(0, 240) + ')' : ''; +}; +const attempts = (m, r) => { + const chain = [r]; + while (chain[0].retry_of) chain.unshift(m.readers.find((x) => x.reader === chain[0].retry_of)); + return chain; +}; +const named = (r) => modelOf(r).model_id + (r.read_only ? ' (read-only retry brief)' : ''); +const listed = (a) => (a.length < 2 ? a.join('') : a.slice(0, -1).join(', ') + ' and ' + a[a.length - 1]); +const retryNote = (m, r, lost) => { + const chain = attempts(m, r); + if (chain.length < 2) return ''; + const refused = chain.filter((x) => String(x.error || '').includes(CYBER)); + const last = chain[chain.length - 1]; + const note = !lost ? 'second-family read by ' + modelOf(last).model_id + ' on the ' + (last.read_only ? 'read-only retry' : 'original') + ' brief after cyber_policy refused ' + listed(refused.map(named)) + : 'second-family read missing: cyber_policy refused ' + listed(refused.map(named)) + (refused.includes(last) ? '' : '; ' + named(last) + ' ' + String(last.error).slice(UNAVAILABLE.length)); + return ',"outside_note":' + JSON.stringify(note); +}; const active = input.stage === 'voices' ? ['voices'] : ['first', 'specialists', 'red-team']; const beforeKey = input.stage === 'voices' ? 'before_voices' : 'before'; @@ -208,8 +239,10 @@ try { if (ended && !outside) { await mark(r.reader, 'failed', { error: t.status }); return stop(r.reader + ' (subagent ' + r.path + ') ' + t.status + '; its review is incomplete, so the pass cannot certify coverage'); } const stuck = async () => r.state_at && (await now()) - Date.parse(r.state_at) > stuckMs; const g = t.status === 'done' || t.status === 'idle' || ended ? await tools.subagent.get({ path: r.path, transcript: 'none' }) : null; + const hostError = outside && ended && g && g.latest_error ? g.latest_error : ''; if (ended && !(g && g.summary)) { - const s = await recordUnavailable(m, r, t.status + ' with no result'); + const refused = hostError ? await refusal(m, r, hostError) : ''; + const s = await recordUnavailable(m, r, refused || t.status + ' with no result'); if (s) return s; continue; } @@ -233,9 +266,10 @@ try { const file = m.out + '/out/' + safe(r.reader) + '.txt'; await tools.capy.write({ path: file, content: g.summary }); if (ended) { - const why = t.status + ' before finishing its review'; + const why = (hostError && cyberWhy(hostError)) || t.status + ' before finishing its review'; const u = await sh(bin + ' collect --ledger ' + q(m.ledger) + roleArgs(r) + ' --output ' + q(file) + ' --merged ' + q(m.merged) + ' --manifest ' + q(mf) + ' --unavailable ' + q(why)); if (u.code === 0) { + if (hostError) await refusal(m, r, hostError); lines.push(u.out.trim()); await mark(r.reader, 'collected', { output: file, error: UNAVAILABLE + why }); continue; @@ -288,7 +322,7 @@ try { const b = await reading(); if (b) { await sh(bin + ' mark --manifest ' + q(mf) + ' --before ' + b.tokens + ',' + b.credits + (beforeKey === 'before_voices' ? ' --voices' : '')); m[beforeKey] = b; } } - if (r.role === 'outside-voice') lines.push(outsideNote(r, r.model || (input.models || {})['outside-voice'] ? 'given' : 'default for an anthropic lane')); + if (r.role === 'outside-voice') lines.push(outsideNote(r, r.retry_of ? 'as R95 retries ' + r.retry_of + ' after its cyber_policy refusal' : r.model || (input.models || {})['outside-voice'] ? 'given' : 'default for an anthropic lane')); if (r.stage === 'voices') { voices = { laneModel: voices.laneModel || input.laneModel || null, readers: { ...voices.readers, [r.reader]: model || null } }; await tools.capy.write({ path: voiceFile, content: JSON.stringify(voices) }); @@ -322,9 +356,9 @@ try { const o = await sh('cat ' + q(r.output)); outs.push('OUTPUT: ' + r.reader + ' file=' + r.output, o.out.trimEnd()); } - for (const r of done.filter((x) => x.role === 'outside-voice')) { + for (const r of done.filter((x) => x.role === 'outside-voice' && !x.superseded_by)) { const lost = String(r.error || '').startsWith(UNAVAILABLE); - outs.push('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":' + JSON.stringify(modelOf(r).model_id) + ',"outside_status":"' + (lost ? 'unavailable' : 'completed') + '" (Step 11\'s review-log rows carry these fields, never the template\'s codex' + outs.push('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":' + JSON.stringify(modelOf(r).model_id) + ',"outside_status":"' + (lost ? 'unavailable' : 'completed') + '"' + retryNote(m, r, lost) + ' (Step 11\'s review-log rows carry these fields, never the template\'s codex' + (lost ? '; first write this voice its own adversarial-review row with "source":"host-subagent","status":"unavailable","completed":false,"converged":false and no --finish, then the native row' : '') + ')'); } if (noteExit !== 0) return finish([...outs, 'STOP: gstack-ship-budget note exited ' + noteExit + ' (its lines above say why); judge nothing further until the budget allows it']); diff --git a/lib/gate-outcomes.ts b/lib/gate-outcomes.ts index 78878821a..5ba6cbc1a 100644 --- a/lib/gate-outcomes.ts +++ b/lib/gate-outcomes.ts @@ -48,6 +48,11 @@ export const GATE_OUTCOMES = { summary: 'the reviewer declined to review', fix: 're-run the review; if it declines again, review the change natively', }, + cyber_policy_refused: { + state: 'unavailable', anchor: 'outside-review-cyber-policy', + summary: 'OpenAI declined the review under its cyber-security policy (cyber_policy)', + fix: "do not send the same brief to another OpenAI model: rerun it on GPT-6 Astra with the read-only correctness brief, or on Gemini 4 Carbon unchanged. Capy's /ship does this once by itself; when every attempt is refused, the second-family read is missing and the PR body says so (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-cyber-policy)", + }, truncated: { state: 'unavailable', anchor: 'outside-review-truncated', summary: 'the reviewer hit its output token limit and the response is cut off', diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index 340c6a604..079cfda04 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -20,6 +20,8 @@ export interface OutsideReviewInput { exit?: number; /** JSONL from `codex exec --json`: command events are positive execution evidence. */ events?: string; + /** The host's own error for the reviewer, such as a Capy subagent's latest_error; never a prompt echo. */ + providerError?: string; } export interface OutsideReviewClassification { @@ -62,6 +64,8 @@ function codexErrorLines(stderr: string): string[] { } /** The account's usage limit: Codex's line carries the reset time, so it is relayed verbatim. Same signature as the probe. */ export const QUOTA_FAILURE = /usage limit|insufficient_quota|exceeded your current quota|quota exceeded/i; +/** OpenAI's cyber-security refusal (R95): Capy relays it as `OpenAI key declined the request: cyber_policy: ...`; read from error lines only. */ +export const CYBER_POLICY_REFUSAL = /\bcyber_policy\b/; /** A plain HTTP 429 or rate limit: transient, so a separate reason from the quota. */ export const RATE_LIMIT_FAILURE = /rate.?limit|too many requests|(?:^|[^0-9])429(?:[^0-9]|$)/i; /** `codex review` transcript on stderr: a command that ran prints " succeeded in Nms:". */ @@ -90,10 +94,14 @@ function execution(input: OutsideReviewInput): OutsideReviewClassification['exec const stderr = input.stderr ?? ''; const exit = input.exit ?? 0; const sandbox = (output: string) => output.match(SANDBOX_FAILURE)?.[0].trim().slice(0, 240); + const cyber = (input.providerError ?? '').split(/\r?\n/).map(line => line.trim()).find(line => CYBER_POLICY_REFUSAL.test(line)); + if (cyber) return { state: 'unavailable', reason: 'cyber_policy_refused', detail: cyber.slice(0, 240) }; if (exit !== 0) { const detail = sandbox(stderr); if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail }; const errors = exit !== 124 ? codexErrorLines(stderr) : []; + const refusal = errors.find(line => CYBER_POLICY_REFUSAL.test(line)); + if (refusal) return { state: 'unavailable', reason: 'cyber_policy_refused', detail: refusal.slice(0, 240) }; const quota = errors.find(line => QUOTA_FAILURE.test(line)); if (quota) return { state: 'unavailable', reason: 'quota_exhausted', detail: quota.slice(0, 240) }; const rateLimit = errors.find(line => RATE_LIMIT_FAILURE.test(line)); @@ -344,7 +352,7 @@ export function validateOutsideReview(text: string, gate: OutsideGate): { comple const VERDICT_EXIT: Record = { clean: 0, findings: 3, unverified: 4, unavailable: 1 }; const GATES = ['review', 'structured', 'spec', 'execution', 'proposal']; const USAGE = `Usage: outside-review-result.ts - outside-review-result.ts --verdict [--stderr ] [--exit ] [--events ] [--label ] + outside-review-result.ts --verdict [--stderr ] [--exit ] [--events ] [--provider-error ] [--label ] Gates: ${GATES.join('|')}. Two-argument form exits 0 completed, 1 unavailable, 2 usage. Verdict form prints VERDICT: clean|findings|unverified|unavailable and exits 0, 3, 4 or 1.`; @@ -361,7 +369,7 @@ if (import.meta.main) { for (let i = 0; i < args.length; i++) { const arg = args[i]!; if (arg === '--verdict') verdictMode = true; - else if (['--stderr', '--exit', '--events', '--label'].includes(arg) && i + 1 < args.length) { flags[arg] = args[++i]!; verdictMode ||= arg !== '--label'; } + else if (['--stderr', '--exit', '--events', '--provider-error', '--label'].includes(arg) && i + 1 < args.length) { flags[arg] = args[++i]!; verdictMode ||= arg !== '--label'; } else if (arg.startsWith('--')) { console.error(USAGE); process.exit(2); } else positional.push(arg); } @@ -380,7 +388,7 @@ if (import.meta.main) { process.exit(0); } const events = await readOptional(flags['--events']); - const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events }); + const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events, providerError: await readOptional(flags['--provider-error']) }); const label = flags['--label'] ?? 'Outside review'; console.log(`VERDICT: ${checked.verdict}`); console.log(`FINDINGS: ${findingsLabel(checked)}`); diff --git a/lib/review-brief.ts b/lib/review-brief.ts index 7e30942c7..ac51a37ce 100644 --- a/lib/review-brief.ts +++ b/lib/review-brief.ts @@ -100,6 +100,14 @@ export function sectionTexts(root: string): SectionTexts { }; } +/** + * The outside voice's read-only correctness brief (R95): sent only when OpenAI refused the original under `cyber_policy`, as + * Capy's pass program retries it on GPT-6 Astra (lib/capy-review-pass.js). It drops the attacker role-play from the outside + * prompt and appends READ_ONLY_RETRY; every other reader, and every first attempt, gets its section text unchanged. + */ +export const READ_ONLY_RETRY = 'How to work, and this overrides anything above that conflicts with it: this is a read-only correctness review of the change, done by reading the code and its tests. Do not role-play an attacker. Do not write, run or describe exploit code or step-by-step reproductions. For each defect give file:line, the condition under which the code returns the wrong answer, and the smallest fix.'; +const READ_ONLY_EDITS: Array<[string, string]> = [['Think like an attacker and a chaos engineer. ', ''], ['Be adversarial. ', ''], ['naming the most exploitable finding', 'naming the most serious finding']]; + const SHA = /^[0-9a-f]{7,40}$/; const FENCE = '`'.repeat(3); function need(slots: Slots, key: string): string { @@ -176,7 +184,8 @@ function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string { try { return `${targetedBrief(need(slots, 'brief'), classes)}\n\n${merged}\n\nApply the targeted brief above (${need(slots, 'checklist')}: ${classes.join(', ')}) in the same read; your reply is this diff's targeted pass.`; } catch (e) { throw e instanceof BriefError ? e : new BriefError((e as Error).message); } } case 'outside-voice': { - const filled = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/ HEAD)', `DIFF_BASE=${diffBase}`]], role); + const outside = slots.read_only ? `${fill(t.outside, READ_ONLY_EDITS, 'read-only retry')}\n\n${READ_ONLY_RETRY}` : t.outside; + const filled = fill(outside, [['DIFF_BASE=$(git merge-base origin/ HEAD)', `DIFF_BASE=${diffBase}`]], role); const body = slots.policy_exclude ? filled.split('git diff "$DIFF_BASE".').join(`git diff "$DIFF_BASE" -- . ${slots.policy_exclude}${slots.keep ? ` and git diff "$DIFF_BASE" -- ${slots.keep}` : ''}.`) : filled; const head = [body, '', ...(slots.disclosure ? [slots.disclosure, ''] : []), `## Branch diff (git diff ${diffBase})`]; if (slots.diff_file) return [...head, `The supplied branch diff is the file ${slots.diff_file} (${need(slots, 'diff_bytes')} bytes, sha256 ${need(slots, 'diff_sha256')}). Read it from disk; it is ${slots.diff_slice ? 'the part of the diff under review' : 'the whole diff under review'}.`, ...(slots.diff_slice ? [slots.diff_slice] : [])].join('\n'); diff --git a/lib/review-pass.ts b/lib/review-pass.ts index 4c8cae78b..36546a160 100644 --- a/lib/review-pass.ts +++ b/lib/review-pass.ts @@ -50,6 +50,8 @@ export interface PassReader { fetch_audit?: { policy_digest: string; events: number; events_sha256: string; hits: FetchHit[]; verdict: 'clean' | 'violation'; enforced: boolean }; /** A stage rerun after a fetch violation (lib/review-sweep.ts): attempt 2 of the slot, and the reader it replaced points at it. */ attempt?: number; superseded_by?: string; + /** An outside voice retried after a cyber_policy refusal (R95, planCyberRetry): the reader it retries, and whether it gets the read-only brief. */ + retry_of?: string; read_only?: true; /** Set when the stage's sweep found a fetch: the reply's new home under /quarantine/; it is never merged, judged or returned. */ quarantined?: { file: string; at: string }; path?: string; output?: string; error?: string; state_at?: string; @@ -221,6 +223,7 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: policy_exclude: r.policy_exclude, ...(m.disclosure ? { disclosure: m.disclosure, keep: r.keep?.map(shellQuote).join(' ') } : {}), ...(m.disclosure && r.excerpt_file ? { excerpt_file: r.excerpt_file.file, excerpt_bytes: String(r.excerpt_file.bytes), excerpt_sha256: r.excerpt_file.sha256, excerpt_files: String(r.excerpt_file.files) } : {}) }; if (role === 'specialist') slots.checklist = path.join(m.checklist_root, 'review/specialists', `${r.name}.md`); + if (r.read_only) slots.read_only = '1'; let findingsSha: string | undefined; if (role === 'red-team') { const findings = specialistFindings(m.merged); @@ -262,6 +265,33 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor: return done; } +/** + * R95 (2026-11-13): after OpenAI refuses the outside voice under cyber_policy, the brief goes once to GPT-6 Astra reworded as a + * read-only correctness review, then once to Gemini 4 Carbon unchanged; never the same brief to another OpenAI model. + */ +export const CYBER_RETRIES: ReadonlyArray<{ name: string; model: PassModel; read_only?: true }> = [ + { name: 'astra-read-only', model: { model_id: 'openai/gpt-6-astra', reasoning_mode: 'high' }, read_only: true }, + { name: 'gemini-4-carbon', model: { model_id: 'google/gemini-4-carbon', reasoning_mode: 'high' } }, +]; + +/** + * Plan the next R95 attempt after `refused`, which the refused reader then points at; null once the chain is spent. A step whose + * model family is in `taken` (the lane's and the native voice's, from the pass's voice-models.json) is skipped, since Step 11 refuses + * an outside voice of either family; with no step left the second-family read is missing. + */ +export function planCyberRetry(m: Manifest, refused: PassReader, taken: ReadonlyArray = []): PassReader | null { + const step = CYBER_RETRIES.slice(CYBER_RETRIES.findIndex(x => x.name === refused.name) + 1).find(x => !taken.includes(x.model.model_id.split('/')[0]!.toLowerCase())); + if (!step || refused.role !== 'outside-voice') return null; + const reader = readerKey('outside-voice', step.name); + const existing = m.readers.find(x => x.reader === reader); + if (existing) return existing.quarantined ? null : existing; + const { file: _f, sha256: _s, bytes: _b, chunks: _c, findings_sha256: _fs, tree: _t, path: _p, output: _o, error: _e, state_at: _a, quarantined: _q, fetch_audit: _fa, attempt: _at, superseded_by: _sb, model: _m, cross_family: _cf, read_only: _ro, ...plan } = refused; + const next: PassReader = { ...plan, reader, name: step.name, title: `ship ${m.pass} ${reader}`, state: 'planned', model: { ...step.model }, retry_of: refused.reader, ...(step.read_only ? { read_only: true as const } : {}) }; + refused.superseded_by = reader; + m.readers.push(next); + return next; +} + export const chunkCount = (text: string) => Math.max(1, Math.ceil(text.length / CHUNK_CHARS)); export function chunkOf(text: string, index: number): string { if (!Number.isInteger(index) || index < 0 || index >= chunkCount(text)) throw new BriefError(`chunk index ${index} is outside 0..${chunkCount(text) - 1}`); diff --git a/test/fixtures/cyber-policy/README.md b/test/fixtures/cyber-policy/README.md new file mode 100644 index 000000000..b79906fa1 --- /dev/null +++ b/test/fixtures/cyber-policy/README.md @@ -0,0 +1,5 @@ +# cyber_policy refusal fixtures + +`capy-latest-error.txt` is the `latest_error` Capy reported, verbatim, for GBRA-392's /ship outside voice +(subagent 7, "ship ship-8903-1-p2 outside-voice", openai/gpt-6.1-sol, status `failed`, 2026-11-13). The same +string came back for every Sol and Astra refusal in the org drive's handoffs/review-refusals-2026-11-13. diff --git a/test/fixtures/cyber-policy/capy-latest-error.txt b/test/fixtures/cyber-policy/capy-latest-error.txt new file mode 100644 index 000000000..d6d2d3c3e --- /dev/null +++ b/test/fixtures/cyber-policy/capy-latest-error.txt @@ -0,0 +1 @@ +OpenAI key declined the request: cyber_policy: This content was flagged for possible cybersecurity risk. If this seems wrong, try rephrasing your request. If you’re doing authorized security work that requires more cyber permissive safeguards, apply for Daybreak access via https://platform.openai.com/settings/organization/status-and-access before retrying. diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 3f78417b2..13a6830af 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -59,6 +59,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'], 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], 'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'], + 'test/fixtures/cyber-policy/**': ['test/outside-review-cyber-policy.test.ts', 'test/review-pass.test.ts'], 'test/fixtures/context-bill/**': ['test/context-bill.test.ts', 'test/context-budget-ratchet.test.ts'], 'test/fixtures/context-budget.json': ['test/helpers/capture-context-budget.ts', 'test/test-value-bar.test.ts'], 'test/fixtures/context-restore-fragment-match-captures.json': ['test/context-restore-fragment-verdict.test.ts'], diff --git a/test/outside-review-cyber-policy.test.ts b/test/outside-review-cyber-policy.test.ts new file mode 100644 index 000000000..3e3b8f606 --- /dev/null +++ b/test/outside-review-cyber-policy.test.ts @@ -0,0 +1,97 @@ +/** + * R95 (2026-11-13): OpenAI refuses some review briefs under its cyber-security policy (`cyber_policy`). That refusal + * is its own outcome, cyber_policy_refused, never a generic failure, so Capy's /ship can retry it on GPT-6 Astra with + * the read-only correctness brief (lib/review-brief.ts) and then Gemini 4 Carbon (lib/capy-review-pass.js). + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { classifyOutsideReview } from '../lib/outside-review-result'; +import { GATE_OUTCOMES, gateOutcomeLine } from '../lib/gate-outcomes'; +import { READ_ONLY_RETRY, renderBrief, sectionTexts } from '../lib/review-brief'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const LIB = path.join(ROOT, 'lib', 'outside-review-result.ts'); +const CAPY_ERROR = fs.readFileSync(path.join(ROOT, 'test/fixtures/cyber-policy/capy-latest-error.txt'), 'utf8'); +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'cyber-policy-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); +const REVIEW = 'Recommendation: ship as-is because no finding survived\nVERDICT: CLEAN'; + +describe('a cyber_policy refusal is its own outcome (R95)', () => { + // Value: protects=the captured Capy refusal (GBRA-392's outside voice) classifies as cyber_policy_refused with the provider's line as detail, so /ship can tell it apart from auth, timeout and empty replies; fails_when=the host error is ignored, or the refusal falls into execution_failed or review_refused; + // why_new=R95; seam=lib/outside-review-result.ts execution(); no_claim=how often OpenAI refuses + test("Capy's captured latest_error is cyber_policy_refused, with the provider's line as detail", () => { + const c = classifyOutsideReview({ text: '', gate: 'review', exit: 1, providerError: CAPY_ERROR }); + expect(c).toMatchObject({ verdict: 'unavailable', reason: 'cyber_policy_refused' }); + expect(c.detail!.startsWith('OpenAI key declined the request: cyber_policy:')).toBe(true); + expect(GATE_OUTCOMES.cyber_policy_refused.state).toBe('unavailable'); + const line = gateOutcomeLine('Outside voice', 'cyber_policy_refused', c.detail); + expect(line).toContain('missing coverage, not a pass'); + expect(line).toContain('docs/troubleshooting.md#outside-review-cyber-policy'); + }); + + // Value: protects=Codex's own trailing ERROR line with cyber_policy is the refusal, while a prompt echo that merely quotes cyber_policy is not; fails_when=the classifier reads the whole stderr (Codex echoes the prompt, and this PR's own diff names cyber_policy); + // why_new=R95; seam=lib/outside-review-result.ts codexErrorLines; no_claim=Codex's exact wording, which is synthetic here + test("Codex's ERROR line is read; a prompt echo that quotes cyber_policy is not", () => { + const echo = 'user\nReview this diff: + if (/cyber_policy/.test(err)) retry()\n'; + const refused = classifyOutsideReview({ text: '', gate: 'review', exit: 1, stderr: echo + 'ERROR: {"error":{"code":"cyber_policy","message":"This content was flagged for possible cybersecurity risk."}}\n' }); + expect(refused.reason).toBe('cyber_policy_refused'); + const auth = classifyOutsideReview({ text: '', gate: 'review', exit: 1, stderr: echo + 'ERROR: 401 Unauthorized: invalid API key\n' }); + expect(auth.reason).toBe('execution_failed'); + }); + + // Value: protects=auth, timeout, empty, quota and plain failures keep their reasons, and a host error without cyber_policy changes nothing; fails_when=the new check swallows another failure or a non-refusal host error starts a retry; + // why_new=R95 (non-refusal failures unchanged); seam=lib/outside-review-result.ts execution(); no_claim=none + test('non-refusal failures keep their reasons', () => { + const cases: Array<[Parameters[0], string]> = [ + [{ text: '', gate: 'review', exit: 124, stderr: 'killed' }, 'timeout'], + [{ text: '', gate: 'review', exit: 0 }, 'empty_response'], + [{ text: '', gate: 'review', exit: 1, stderr: 'ERROR: You exceeded your current quota\n' }, 'quota_exhausted'], + [{ text: '', gate: 'review', exit: 1, stderr: 'ERROR: 401 Unauthorized\n' }, 'execution_failed'], + [{ text: '', gate: 'review', exit: 1, providerError: 'OpenAI key declined the request: invalid_api_key' }, 'execution_failed'], + [{ text: 'I cannot review this change.', gate: 'review', exit: 0 }, 'review_refused'], + ]; + for (const [input, reason] of cases) expect(classifyOutsideReview(input).reason).toBe(reason as never); + }); + + // Value: protects=a completed review that discusses cyber_policy, with or without cyber_policy on stderr after a clean exit, still counts as the review it is, while the host's own error (providerError) naming cyber_policy is the refusal whatever the reply says; fails_when=any mention of cyber_policy in the reply or on stderr makes it a refusal, or a host refusal is read as a clean review; + // why_new=R95 negative control; seam=lib/outside-review-result.ts; no_claim=none + test('a normal review is never a refusal', () => { + const text = 'The new check matches cyber_policy in the error.\n' + REVIEW; + expect(classifyOutsideReview({ text, gate: 'review', exit: 0 })).toMatchObject({ verdict: 'clean' }); + expect(classifyOutsideReview({ text, gate: 'review', exit: 0, stderr: 'cyber_policy\n' })).toMatchObject({ verdict: 'clean' }); + expect(classifyOutsideReview({ text, gate: 'review', exit: 0, providerError: CAPY_ERROR }).reason).toBe('cyber_policy_refused'); + }); + + // Value: protects=the CLI the Capy pass program calls reads --provider-error and prints REASON: cyber_policy_refused; fails_when=the flag is refused as usage or ignored; + // why_new=R95; seam=outside-review-result.ts --verdict; no_claim=none + test('the --verdict CLI reads --provider-error', () => { + const err = path.join(TMP, 'err.txt'), reply = path.join(TMP, 'reply.txt'); + fs.writeFileSync(err, CAPY_ERROR); + fs.writeFileSync(reply, ''); + const r = spawnSync(process.execPath, [LIB, '--verdict', '--provider-error', err, '--exit', '1', '--label', 'Outside voice', 'review', reply], { encoding: 'utf8', timeout: 10_000 }); + expect(r.status).toBe(1); + expect(r.stdout).toContain('REASON: cyber_policy_refused'); + expect(r.stderr).toContain('Outside voice unavailable: OpenAI declined the review under its cyber-security policy (cyber_policy)'); + }); +}); + +describe('the read-only retry brief (R95)', () => { + const t = sectionTexts(ROOT); + const slots = { diff_base: 'abc1234', diff: '+x' }; + // Value: protects=the retry brief drops attacker role-play, asks for file:line, the wrong-answer condition and the smallest fix, and keeps every marker collect needs, while the default outside brief is unchanged; fails_when=the retry text leaks into the default brief, or the retry loses the Recommendation/VERDICT markers; + // why_new=R95 (reworded only on the retry); seam=lib/review-brief.ts outside-voice; no_claim=that the retry finds what the original would + test('only the retry render is reworded, and it keeps the reply markers', () => { + const plain = renderBrief('outside-voice', slots, t); + const retry = renderBrief('outside-voice', { ...slots, read_only: '1' }, t); + expect(plain).toContain('Think like an attacker and a chaos engineer.'); + expect(plain).not.toContain(READ_ONLY_RETRY); + expect(retry).toContain(READ_ONLY_RETRY); + for (const gone of ['Think like an attacker', 'Be adversarial.', 'most exploitable']) expect(retry).not.toContain(gone); + for (const kept of ['Severity: Critical|High|Medium|Low', 'Recommendation: because', 'VERDICT: CLEAN', 'git diff "$DIFF_BASE"']) expect(retry).toContain(kept); + expect(READ_ONLY_RETRY).toContain('For each defect give file:line, the condition under which the code returns the wrong answer, and the smallest fix'); + expect(() => renderBrief('outside-voice', { ...slots, read_only: '1' }, { ...t, outside: t.outside.replace('Be adversarial. ', '') })).toThrow(/placeholder Be adversarial/); + }); +}); diff --git a/test/review-pass.test.ts b/test/review-pass.test.ts index f7f52a3e8..33badb2da 100644 --- a/test/review-pass.test.ts +++ b/test/review-pass.test.ts @@ -7,8 +7,8 @@ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { promptHash, sectionTexts, withWorkspace, workspaceParagraph, workspaceStep } from '../lib/review-brief'; -import { CHUNK_CHARS, PARENT_ONLY_QA, buildManifest, chunkCount, chunkOf, markReader, renderStage, specialistFindings, type Manifest } from '../lib/review-pass'; +import { READ_ONLY_RETRY, promptHash, sectionTexts, withWorkspace, workspaceParagraph, workspaceStep } from '../lib/review-brief'; +import { CHUNK_CHARS, PARENT_ONLY_QA, buildManifest, chunkCount, chunkOf, markReader, planCyberRetry, renderStage, specialistFindings, type Manifest } from '../lib/review-pass'; import type { Dispatch } from '../lib/review-depth'; import { FIXTURE_MARKER } from '../lib/fixture-marker'; @@ -137,7 +137,7 @@ function repo(): { dir: string; env: NodeJS.ProcessEnv } { return { dir, env }; } -interface Sub { title: string; prompt: string; status: string; polls: number; model?: unknown; summary?: string; late?: boolean } +interface Sub { title: string; prompt: string; status: string; polls: number; model?: unknown; summary?: string; late?: boolean; error?: string } /** A fake Capy host: the shell is real (the repository's bins), subagents are scripted. */ const cleanReply = (title: string) => (title.endsWith('outside-voice') ? 'Recommendation: ship as-is because no finding survived' : 'NO FINDINGS'); /** The REVIEWED_TREE line a reader prints from dir (its brief's checkout when it follows the workspace paragraph); '' outside a checkout. */ @@ -148,7 +148,7 @@ function treeLine(dir: string): string { return tree.status === 0 && top.status === 0 ? `REVIEWED_TREE: ${tree.stdout.trim()} ${top.stdout.trim()}\n` : ''; } const briefDir = (prompt: string) => /Workspace: the checkout under review is (.+?)\. Your shell and file tools/.exec(prompt)?.[1] ?? ''; -function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) => string; reportFrom?: (title: string) => string; fail?: (title: string) => boolean; reject?: (title: string) => boolean; createThrows?: (title: string) => 'before' | 'after' | undefined; pollsToFinish?: number; lateSummary?: boolean; hang?: (title: string) => boolean; stopped?: (title: string) => boolean; keepSummary?: boolean; epoch?: number } = {}) { +function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) => string; reportFrom?: (title: string) => string; fail?: (title: string) => boolean; reject?: (title: string) => boolean; createThrows?: (title: string) => 'before' | 'after' | undefined; pollsToFinish?: number; lateSummary?: boolean; hang?: (title: string) => boolean; stopped?: (title: string) => boolean; keepSummary?: boolean; epoch?: number; error?: (title: string) => string | undefined } = {}) { let clock = 0; const subs = new Map(); const creates: Sub[] = []; @@ -160,7 +160,8 @@ function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) if (s.late && !s.summary) { s.summary = reply(s); continue; } if (s.status !== 'working' || o.hang?.(s.title)) continue; if (++s.polls < (o.pollsToFinish ?? 1)) continue; - if (o.stopped?.(s.title)) { s.status = 'stopped'; if (o.keepSummary) s.summary = reply(s); } + if (o.error?.(s.title)) { s.status = 'failed'; s.error = o.error(s.title); if (o.keepSummary) s.summary = reply(s); } + else if (o.stopped?.(s.title)) { s.status = 'stopped'; if (o.keepSummary) s.summary = reply(s); } else if (o.fail?.(s.title)) { s.status = 'failed'; if (o.keepSummary) s.summary = reply(s); } else if (o.lateSummary && !s.late) { s.status = 'done'; s.late = true; } else { s.status = 'done'; s.summary = reply(s); } @@ -190,7 +191,7 @@ function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) }, start: async ({ indices, model }: { indices: number[]; model?: unknown }) => { starts.push(indices); for (const i of indices) { const s = subs.get(String(i))!; s.status = 'working'; if (model) s.model = model; } return { started: indices.map(index => ({ index })), rejected: [] }; }, list: async () => { advance(); return { tasks: [...subs].map(([p, s]) => ({ path: p, title: s.title, status: s.status })), total: subs.size }; }, - get: async ({ path: p }: { path: string }) => ({ path: p, status: subs.get(p)!.status, summary: subs.get(p)!.summary }), + get: async ({ path: p }: { path: string }) => ({ path: p, status: subs.get(p)!.status, summary: subs.get(p)!.summary, ...(subs.get(p)!.error ? { latest_error: subs.get(p)!.error } : {}) }), stop: async ({ paths }: { paths: string[] }) => { stops.push(...paths); for (const p of paths) subs.get(p)!.status = 'stopped'; return { results: paths.map(p => ({ path: p, ok: true })) }; }, }, thread: { get: async () => ({ cost: { tokens: 1000 + clock, credits: 1 + clock / 1000 } }) }, @@ -630,6 +631,133 @@ describe('Capy pass program (fake host, real bins)', () => { expect(v.stdout).not.toContain('NOT_REVIEWED'); }, 300_000); + // R95 (2026-11-13): a cyber_policy refusal of the outside voice is retried once on GPT-6 Astra with the read-only brief, then on Gemini 4 Carbon with the original brief; when every attempt fails the second-family read is missing, disclosed, and Step 11 goes on. + const CYBER = fs.readFileSync(path.join(ROOT, 'test/fixtures/cyber-policy/capy-latest-error.txt'), 'utf8').trim(); + const voiceReply = (t: string) => (t.includes('outside-voice') ? 'Recommendation: ship as-is because no finding survived' : cleanReply(t)); + async function refusedVoices(pass: string, error: (title: string) => string | undefined, lane: Record = {}) { + const { dir, env } = repo(); + const out = path.join(dir, '.state/run/p1'); + const h = host(dir, env, { error, output: voiceReply, epoch: Date.now() }); + const base = { base: 'main', out, pass, planFlags: ['--depth-floor', 'full'], ...lane }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + let res = '', all = ''; + for (let i = 0; i < 12; i++) { + res = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 }); + all += res + '\n'; + if (!res.includes('NEXT: poll')) break; + } + const created = (suffix: string) => h.creates.find(c => c.title === 'ship ' + pass + ' ' + suffix); + const verify = bin(dir, env, ['verify', '--ledger', path.join(out, 'review-ledger.jsonl')]).stdout; + return { res, all, h, created, verify, base }; + } + + // Value: protects=R95 on the real pass program: Capy's captured cyber_policy refusal of the Sol outside voice is recorded unavailable with its own reason and retried once on openai/gpt-6-astra with the read-only brief, whose review is the Step 11 outside record; fails_when=the refusal is recorded as a plain failure with no retry, the retry resends the original brief, or the original's unavailable record replaces the retry's; + // why_new=R95; seam=lib/capy-review-pass.js ended outside voice; no_claim=that Astra finishes a real refused brief (one live smoke in GSTA-537) + test('a cyber_policy refusal is retried once on Astra with the read-only brief (R95)', async () => { + const { res, all, h, created, verify } = await refusedVoices('x40p1', t => (t.endsWith(' outside-voice') ? CYBER : undefined)); + expect(res).not.toContain('STOP:'); + expect(all).toContain('RETRY: reader=outside-voice/astra-read-only model=openai/gpt-6-astra brief=read-only'); + expect(all).toMatch(/COLLECT_UNAVAILABLE: reader=outside-voice .*/); + const astra = created('outside-voice/astra-read-only')!; + expect(astra.model).toEqual({ model_id: 'openai/gpt-6-astra', reasoning_mode: 'high' }); + expect(astra.prompt).toContain(READ_ONLY_RETRY); + expect(astra.prompt).not.toContain('Think like an attacker'); + expect(created('outside-voice')!.prompt).toContain('Think like an attacker and a chaos engineer.'); + expect(created('outside-voice/gemini-4-carbon')).toBeUndefined(); + expect(res).toContain('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":"openai/gpt-6-astra","outside_status":"completed","outside_note":"second-family read by openai/gpt-6-astra on the read-only retry brief after cyber_policy refused openai/gpt-6.1-sol"'); + expect(res).not.toContain('"outside_model":"openai/gpt-6.1-sol"'); + expect(res).toContain('NEXT: judge the voices'); + expect(verify).toMatch(/^BRIEF outside-voice UNAVAILABLE/m); + expect(verify).toMatch(/^OUTPUT outside-voice UNAVAILABLE .*reason=".*cyber_policy/m); + expect(verify).toMatch(/^BRIEF outside-voice\/astra-read-only MATCH/m); + expect(verify).not.toContain('MISMATCH'); + expect(verify).toMatch(/^VERIFY: .* unavailable=1$/m); + expect(h.creates.filter(c => c.title.includes('outside-voice')).length).toBe(2); + }, 600_000); + + // Value: protects=an Astra refusal of the retry falls through to google/gemini-4-carbon with the original brief byte for byte, and a refusal or failure there records the second-family read as missing (a disclosed gap, not a pass) while Step 11 goes on to the native voice; fails_when=a fourth attempt starts, Gemini gets the reworded brief, or the exhausted chain STOPs the pass; + // why_new=R95; seam=lib/capy-review-pass.js retry chain; no_claim=Gemini's availability under another organization's model policy + test('an Astra refusal falls through to Gemini 4 Carbon with the original brief, then the second-family read is missing (R95)', async () => { + const { res, all, h, created, verify } = await refusedVoices('x41p1', t => (/outside-voice(\/astra-read-only|\/gemini-4-carbon)?$/.test(t) ? CYBER : undefined)); + expect(res).not.toContain('STOP:'); + expect(all).toContain('RETRY: reader=outside-voice/astra-read-only model=openai/gpt-6-astra brief=read-only'); + expect(all).toContain('RETRY: reader=outside-voice/gemini-4-carbon model=google/gemini-4-carbon brief=original'); + expect(all).toContain('RETRY: exhausted'); + const gemini = created('outside-voice/gemini-4-carbon')!; + expect(gemini.model).toEqual({ model_id: 'google/gemini-4-carbon', reasoning_mode: 'high' }); + expect(gemini.prompt).toBe(created('outside-voice')!.prompt); + expect(h.creates.filter(c => c.title.includes('outside-voice')).length).toBe(3); + expect(res).toContain('"outside_model":"google/gemini-4-carbon","outside_status":"unavailable","outside_note":"second-family read missing: cyber_policy refused openai/gpt-6.1-sol, openai/gpt-6-astra (read-only retry brief) and google/gemini-4-carbon"'); + expect(res).toContain('first write this voice its own adversarial-review row'); + expect(res).toContain('NEXT: judge the voices'); + expect(verify).not.toContain('MISMATCH'); + expect(verify).toMatch(/^BRIEF outside-voice\/gemini-4-carbon UNAVAILABLE/m); + expect(verify).toMatch(/^VERIFY: .* unavailable=3$/m); + }, 900_000); + + // Value: protects=on a Google-family lane the Gemini step is skipped (Step 11 refuses an outside voice of the lane's family), so two OpenAI refusals record the second-family read as missing and a resume finishes instead of stopping; fails_when=Gemini is planned beside a Google lane and familyStop stops every resume; + // why_new=GSTA-537 /ship outside-voice finding (lib/review-pass.ts:274), fixed under the Captain's scoped fix-check grant; seam=planCyberRetry via gstack-review-dispatch retry; no_claim=which outside voice a Google lane should use + test('a Google-family lane skips the Gemini retry and records the second-family read missing, with no STOP on resume (R95)', async () => { + const lane = { laneModel: 'google/gemini-4-argon', models: { 'outside-voice': { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'medium' } } }; + const { res, all, h, created, base } = await refusedVoices('x45p1', t => (/outside-voice(\/astra-read-only)?$/.test(t) ? CYBER : undefined), lane); + expect(all).not.toContain('STOP:'); + expect(all).toContain('RETRY: reader=outside-voice/astra-read-only model=openai/gpt-6-astra brief=read-only'); + expect(all).toContain('RETRY: exhausted reader=outside-voice/astra-read-only'); + expect(created('outside-voice/gemini-4-carbon')).toBeUndefined(); + expect(res).toContain('"outside_status":"unavailable","outside_note":"second-family read missing: cyber_policy refused openai/gpt-6.1-sol and openai/gpt-6-astra (read-only retry brief)"'); + expect(res).toContain('NEXT: judge the voices'); + const again = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 }); + expect(again).not.toContain('STOP:'); + }, 600_000); + + // Value: protects=planCyberRetry skips only the steps whose family the lane or native voice holds: an Anthropic lane still gets Astra then Gemini, a Google lane gets Astra and then nothing; fails_when=the family check drops Gemini for every lane or keeps it beside a Google lane; + // why_new=GSTA-537 scoped fix-check (control for the Google-lane case); seam=lib/review-pass.ts planCyberRetry; no_claim=none + test('planCyberRetry keeps Gemini for an Anthropic lane and skips it for a Google lane (R95)', () => { + const chain = (taken: string[]) => { + const m = { pass: 'u1', readers: [{ reader: 'outside-voice', role: 'outside-voice', stage: 'voices', state: 'collected' }] } as unknown as Manifest; + const astra = planCyberRetry(m, m.readers[0]!, taken); + return [astra?.reader, astra ? planCyberRetry(m, astra, taken)?.reader ?? null : null]; + }; + expect(chain(['anthropic'])).toEqual(['outside-voice/astra-read-only', 'outside-voice/gemini-4-carbon']); + expect(chain(['google'])).toEqual(['outside-voice/astra-read-only', null]); + }); + + // Value: protects=only cyber_policy starts a retry: a failed voice whose host error is another provider failure, and a voice that reviews normally, each run exactly one outside voice; fails_when=any failure, or a success, dispatches an Astra or Gemini reader; + // why_new=R95 (no new retries for anything else; negative control); seam=lib/capy-review-pass.js; no_claim=none + test('a non-refusal failure and a normal review never retry (R95)', async () => { + const failed = await refusedVoices('x42p1', t => (t.endsWith(' outside-voice') ? 'OpenAI key declined the request: invalid_api_key: Incorrect API key provided.' : undefined)); + expect(failed.res).not.toContain('STOP:'); + expect(failed.all).toContain('RETRY: none'); + expect(failed.res).toContain('"outside_model":"openai/gpt-6.1-sol","outside_status":"unavailable"'); + expect(failed.h.creates.filter(c => c.title.includes('outside-voice')).length).toBe(1); + const clean = await refusedVoices('x43p1', () => undefined); + expect(clean.all).not.toContain('RETRY:'); + expect(clean.res).toContain('"outside_model":"openai/gpt-6.1-sol","outside_status":"completed"'); + expect(clean.res).not.toContain('outside_note'); + expect(clean.h.creates.filter(c => c.title.includes('outside-voice')).length).toBe(1); + }, 600_000); + + // Value: protects=a voice that the host marks refused under cyber_policy but that still wrote a review is collected as that review, with its own outside record, and starts no retry; fails_when=the retry is planned before the program knows the voice left findings, so a redundant Astra reader runs and the collected voice's OUTSIDE_RECORD is dropped; + // why_new=GSTA-537 /ship red-team finding (capy-review-pass.js refusal before collect); seam=lib/capy-review-pass.js ended outside voice; no_claim=how often a refused voice leaves a review + test('a cyber_policy refusal that still left a review is collected and never retried (R95)', async () => { + const { dir, env } = repo(); + const out = path.join(dir, '.state/run/p1'); + const h = host(dir, env, { error: t => (t.endsWith(' outside-voice') ? CYBER : undefined), keepSummary: true, output: t => (t.endsWith('outside-voice') ? '1. **A guard fails open** - `src/a.ts:1` accepts any input. Severity: Medium. Blocks: no.\n\nRecommendation: fix the guard because it fails open' : cleanReply(t)), epoch: Date.now() }); + const base = { base: 'main', out, pass: 'x44p1', planFlags: ['--depth-floor', 'full'] }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + let res = '', all = ''; + for (let i = 0; i < 6; i++) { + res = await runPass(h, { ...base, stage: 'voices', stuckMs: 60_000 }); + all += res + '\n'; + if (!res.includes('NEXT: poll')) break; + } + expect(res).not.toContain('STOP:'); + expect(all).not.toContain('RETRY:'); + expect(h.creates.filter(c => c.title.includes('outside-voice')).length).toBe(1); + expect(res).toContain('OUTSIDE_RECORD: "outside_provider":"host-subagent","outside_model":"openai/gpt-6.1-sol","outside_status":"completed"'); + expect(fs.readFileSync(path.join(out, 'merged.jsonl'), 'utf8')).toContain('A guard fails open'); + }, 300_000); + test('an audit reader marked first settles before any specialist starts; its output is returned, never merged', async () => { const { dir, env } = repo(); const out = path.join(dir, '.state/run/p1');