diff --git a/lib/capy-review-pass.js b/lib/capy-review-pass.js index aba3ee5de..cf7add115 100644 --- a/lib/capy-review-pass.js +++ b/lib/capy-review-pass.js @@ -74,8 +74,30 @@ const briefText = async (r) => { }; const roleArgs = (r) => ' --role ' + q(r.role) + (r.name ? ' --name ' + q(r.name) : ''); const REASONING = { medium: 'medium', light: 'low' }; +const family = (id) => (typeof id === 'string' && id.includes('/') ? id.slice(0, id.indexOf('/')) : ''); +const CERTIFIED_OUTSIDE = { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'medium' }; +const laneFamily = family(input.laneModel); const modelOf = (r) => r.model || (input.models || {})[r.role === 'specialist-batched' ? 'specialist' : r.role] + || (r.role === 'outside-voice' && laneFamily === 'anthropic' ? CERTIFIED_OUTSIDE : undefined) || (REASONING[r.effort] && input.laneModel ? { model_id: input.laneModel, reasoning_mode: REASONING[r.effort] } : undefined); +const outsideNote = (r) => { + const ov = modelOf(r); + const certified = laneFamily === 'anthropic' && ov.model_id === CERTIFIED_OUTSIDE.model_id && ov.reasoning_mode === CERTIFIED_OUTSIDE.reasoning_mode; + return 'OUTSIDE_VOICE: ' + ov.model_id + ' reasoning=' + (ov.reasoning_mode || 'default') + ((input.models || {})['outside-voice'] ? ' given' : ' default for an anthropic lane') + + (certified ? ' (certified configuration)' : '; outside the certified configuration (' + CERTIFIED_OUTSIDE.model_id + ' at ' + CERTIFIED_OUTSIDE.reasoning_mode + ' beside an anthropic lane)'); +}; +const familyStop = (readers) => { + const ov = readers.find((x) => x.role === 'outside-voice' && !['created', 'collected', 'failed'].includes(x.state)); + if (!ov) return null; + const om = modelOf(ov); + if (!om) return "the outside voice needs input.models['outside-voice'] (a model of another family); none was given"; + const native = readers.find((x) => x.role === 'adversarial-native'); + const nativeFamily = family(native && modelOf(native) ? modelOf(native).model_id : undefined); + if (!laneFamily && !nativeFamily) return 'the outside voice ' + om.model_id + ' cannot be checked against another family: neither input.laneModel nor a native voice model was given'; + const f = family(om.model_id); + if (f !== laneFamily && f !== nativeFamily) return null; + return 'the outside voice ' + om.model_id + ' is the ' + f + ' family, the same as the ' + (f === nativeFamily ? 'native voice' : 'lane model') + "; Step 11 needs input.models['outside-voice'] from another family"; +}; const safe = (s) => s.replace(/[^a-zA-Z0-9._-]/g, '_'); const active = input.stage === 'voices' ? ['voices'] : ['first', 'specialists', 'red-team']; @@ -93,6 +115,8 @@ try { lines.push('REVIEW_LOG_START: ' + s + ' token=' + token); } for (;;) { + const sameFamily = input.stage === 'voices' && familyStop(m.readers); + if (sameFamily) return stop(sameFamily); const failed = m.readers.find((x) => active.includes(x.stage) && x.state === 'failed'); if (failed) return stop(failed.reader + ' (subagent ' + failed.path + ') ' + (failed.error || 'failed') + '; its review is incomplete, so the pass cannot certify coverage; restore it with a fresh pass under a new --pass and --out'); const list = await tools.subagent.list({}); @@ -145,7 +169,6 @@ try { lines.push(...s.out.trim().split('\n')); m = await load(); } - if (m.readers.some((x) => x.stage === stage && x.role === 'outside-voice' && x.state === 'rendered' && !modelOf(x))) return stop("the outside voice needs input.models['outside-voice'] (a model of another family); none was given"); const unmapped = m.readers.find((x) => x.stage === stage && x.state === 'rendered' && REASONING[x.effort] && !modelOf(x)); if (unmapped) return stop(unmapped.reader + ' runs at effort=' + unmapped.effort + ', which needs input.laneModel (your model id) or input.models; none was given'); for (const r of m.readers.filter((x) => x.stage === stage && x.state === 'rendered')) { @@ -162,6 +185,7 @@ try { const b = await reading(); if (b) { await sh(bin + ' mark --manifest ' + q(mf) + ' --before ' + b.tokens + ',' + b.credits); m.before = b; } } + if (r.role === 'outside-voice') lines.push(outsideNote(r)); await mark(r.reader, 'intent', {}); const res = await tools.subagent.create({ tasks: [{ title: r.title, prompt: text }], start: { machine: 'shared', ...(model ? { model } : {}) } }); const idx = String(res.tasks[0].index); diff --git a/ship/DRIVER.md b/ship/DRIVER.md index bea60744f..2b1bb361b 100644 --- a/ship/DRIVER.md +++ b/ship/DRIVER.md @@ -25,11 +25,11 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** ( outside-voice and documentation brief, its severity and confidence rules and its gates come from its section verbatim, with only its placeholders filled; this file never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render` -file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a -small model at low effort; their findings go to the PR body. +file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs at low effort on a small model, +Step 14.5 once on Sonnet 5.5; findings go to the PR body. **Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's -section read or a fix-check, plus a keep-alive per 200 s waited. A fix repeats Phases 3-4. +section read or a fix-check, plus a keep-alive per 200 s waited. **Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends with `gstack-ship-budget note --step --reading R --credits C --record `, R and C read in that program diff --git a/ship/DRIVER.md.tmpl b/ship/DRIVER.md.tmpl index 2751871bf..a7c675c27 100644 --- a/ship/DRIVER.md.tmpl +++ b/ship/DRIVER.md.tmpl @@ -23,11 +23,11 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** ( outside-voice and documentation brief, its severity and confidence rules and its gates come from its section verbatim, with only its placeholders filled; this file never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render` -file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a -small model at low effort; their findings go to the PR body. +file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs at low effort on a small model, +Step 14.5 once on Sonnet 5.5; findings go to the PR body. **Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's -section read or a fix-check, plus a keep-alive per 200 s waited. A fix repeats Phases 3-4. +section read or a fix-check, plus a keep-alive per 200 s waited. **Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends with `gstack-ship-budget note --step --reading R --credits C --record `, R and C read in that program diff --git a/test/review-pass.test.ts b/test/review-pass.test.ts index 9bfc99c89..f320ff1b3 100644 --- a/test/review-pass.test.ts +++ b/test/review-pass.test.ts @@ -303,7 +303,7 @@ describe('Capy pass program (fake host, real bins)', () => { const h = host(dir, env); const base = { base: 'main', out, pass: 'x7p1', planFlags: ['--depth-floor', 'full'] }; expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); - expect(await runPass(h, { ...base, stage: 'voices' })).toContain("STOP: the outside voice needs input.models['outside-voice']"); + expect(await runPass(h, { ...base, stage: 'voices', laneModel: undefined })).toContain("STOP: the outside voice needs input.models['outside-voice']"); expect(h.creates.some(c => c.title.startsWith('ship x7p1 ') && /adversarial|outside/.test(c.title))).toBe(false); const models = { 'outside-voice': { model_id: 'openai/gpt-6.1-sol' }, 'adversarial-native': { model_id: 'anthropic/claude-opus-5-5', reasoning_mode: 'high' } }; expect(await runPass(h, { ...base, stage: 'voices', models })).toContain('NEXT: judge the voices'); @@ -311,6 +311,99 @@ describe('Capy pass program (fake host, real bins)', () => { expect(h.creates.find(c => c.title === 'ship x7p1 adversarial-native')!.model).toEqual(models['adversarial-native']); }, 240_000); + // Value: protects=Step 11's outside voice is never the native voice's or the lane's model family; fails_when=the STOP checks only that some model was given (lib/capy-review-pass.js before GSTA-167 PR B), so a Claude outside voice beside a Claude native voice runs as "outside"; + // why_new=GSTA-167 plan R2 (autoplan-20261031135232); seam=none; no_claim=which family pairs find more + test('a same-family explicit outside voice STOPs before either voice is created', async () => { + const { dir, env } = repo(); + const h = host(dir, env); + const base = { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x14p1', planFlags: ['--depth-floor', 'full'] }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + expect(await runPass(h, { ...base, stage: 'voices', models: { 'outside-voice': { model_id: 'anthropic/claude-sonnet-5-5', reasoning_mode: 'medium' } } })) + .toContain("STOP: the outside voice anthropic/claude-sonnet-5-5 is the anthropic family, the same as the lane model; Step 11 needs input.models['outside-voice'] from another family"); + expect(await runPass(h, { ...base, stage: 'voices', models: { 'outside-voice': { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'medium' }, 'adversarial-native': { model_id: 'openai/gpt-6-astra', reasoning_mode: 'high' } } })) + .toContain('STOP: the outside voice openai/gpt-6.1-sol is the openai family, the same as the native voice'); + expect(h.creates.some(c => /^ship x14p1 (adversarial-native|outside-voice)$/.test(c.title))).toBe(false); + }, 240_000); + + // Value: protects=a Step 11 rerun on one manifest keeps refusing a same-family outside voice, including one queued at the subagent cap that tools.subagent.start would pick up; fails_when=the family check runs only on the first render, so a resumed or queued voice starts unchecked; + // why_new=GSTA-167 plan R2; seam=none; no_claim=the host's cap timing + test('a Step 11 rerun on one manifest still refuses a same-family outside voice, rendered or queued', async () => { + const same = { 'outside-voice': { model_id: 'anthropic/claude-sonnet-5-5', reasoning_mode: 'medium' } }; + const refused = /STOP: the outside voice anthropic\/claude-sonnet-5-5 is the anthropic family/; + const { dir, env } = repo(); + const h = host(dir, env); + const base = { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x15p1', planFlags: ['--depth-floor', 'full'] }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + expect(await runPass(h, { ...base, stage: 'voices', models: same })).toMatch(refused); + expect(await runPass(h, { ...base, stage: 'voices', models: same })).toMatch(refused); + expect(h.creates.some(c => /^ship x15p1 (adversarial-native|outside-voice)$/.test(c.title))).toBe(false); + + const b = repo(); + const h2 = host(b.dir, b.env, { reject: t => t.endsWith('outside-voice') }); + const base2 = { ...base, out: path.join(b.dir, '.state/run/p1'), pass: 'x18p1' }; + expect(await runPass(h2, base2)).toContain('NEXT: judge (Phase 4)'); + expect(await runPass(h2, { ...base2, stage: 'voices', budgetMs: 1 })).toMatch(/DISPATCHED: outside-voice path=\d+ queued/); + const starts = h2.starts.length; + expect(await runPass(h2, { ...base2, stage: 'voices', models: same })).toMatch(refused); + expect(h2.starts.length).toBe(starts); + }, 300_000); + + // Value: protects=a lane whose family is unknown or not Anthropic gets no default outside voice, and an explicit one cannot skip the family check; fails_when=the pin applies to a lane it was not certified for, or an outside voice runs with nothing to compare its family to; + // why_new=GSTA-167 plan R2; seam=none; no_claim=an OpenAI lane's pairing, which is uncertified + test('a missing lane identity STOPs the outside voice: none given, or given with nothing to compare its family to', async () => { + const { dir, env } = repo(); + const h = host(dir, env); + const base = { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x16p1', planFlags: ['--depth-floor', 'full'] }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + const none = "STOP: the outside voice needs input.models['outside-voice'] (a model of another family); none was given"; + expect(await runPass(h, { ...base, stage: 'voices', laneModel: undefined })).toContain(none); + expect(await runPass(h, { ...base, stage: 'voices', laneModel: 'openai/gpt-6.1-sol' })).toContain(none); + expect(await runPass(h, { ...base, stage: 'voices', laneModel: undefined, models: { 'outside-voice': { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'medium' } } })) + .toContain('STOP: the outside voice openai/gpt-6.1-sol cannot be checked against another family: neither input.laneModel nor a native voice model was given'); + expect(h.creates.some(c => /^ship x16p1 (adversarial-native|outside-voice)$/.test(c.title))).toBe(false); + }, 240_000); + + const certified = { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'medium' }; + async function voicesRun(dir: string, env: NodeJS.ProcessEnv, sub: string, models?: Record) { + const out = path.join(dir, '.state/run', sub); + const h = host(dir, env); + const base = { base: 'main', out, pass: 'x17p1', planFlags: ['--depth-floor', 'full'] }; + expect(await runPass(h, base)).toContain('NEXT: judge (Phase 4)'); + const res = await runPass(h, { ...base, stage: 'voices', ...(models ? { models } : {}) }); + const m = JSON.parse(fs.readFileSync(path.join(out, 'pass-manifest.json'), 'utf8')) as Manifest; + for (const r of m.readers) expect(h.creates.find(c => c.title === r.title)!.prompt).toBe(fs.readFileSync(r.file!, 'utf8')); + const shas = Object.fromEntries(h.creates.map(c => [c.title, promptHash(c.prompt.split(out).join(''))])); + return { res, shas, models: Object.fromEntries(h.creates.map(c => [c.title, c.model])) }; + } + + // Value: protects=an Anthropic lane's Step 11 gets the certified outside voice (openai/gpt-6.1-sol at medium, as rounds 3-6 and A3668 ran) without naming it, and every brief goes out byte for byte as it does with that model named; fails_when=the lane must pick the outside voice (gpt-6-astra in 6c/7e/8g), or the pin changes any reader's brief or another reader's model; + // why_new=GSTA-167 plan R2; seam=none; no_claim=the certified pair's recall, which arm-L6 measured + test('an anthropic lane defaults the outside voice to the certified model, with every brief byte-identical to naming it', async () => { + const { dir, env } = repo(); + const pinned = await voicesRun(dir, env, 'pin'); + expect(pinned.res).toContain('OUTSIDE_VOICE: openai/gpt-6.1-sol reasoning=medium default for an anthropic lane (certified configuration)'); + expect(pinned.res).toContain('NEXT: judge the voices'); + expect(pinned.models['ship x17p1 outside-voice']).toEqual(certified); + const named = await voicesRun(dir, env, 'named', { 'outside-voice': certified }); + expect(named.res).toContain('OUTSIDE_VOICE: openai/gpt-6.1-sol reasoning=medium given (certified configuration)'); + expect(named.shas).toEqual(pinned.shas); + expect(named.models).toEqual(pinned.models); + }, 300_000); + + // Value: protects=an explicit cross-family override is accepted but its pass output says it is outside the certified configuration; fails_when=a changed outside-voice model or reasoning passes silently as if certified; + // why_new=GSTA-167 plan R2; seam=none; no_claim=the uncertified configuration's recall + test('an explicit outside-voice override with changed reasoning runs and is recorded as uncertified, other briefs unchanged', async () => { + const { dir, env } = repo(); + const high = { model_id: 'openai/gpt-6.1-sol', reasoning_mode: 'high' }; + const over = await voicesRun(dir, env, 'over', { 'outside-voice': high }); + expect(over.res).toContain('OUTSIDE_VOICE: openai/gpt-6.1-sol reasoning=high given; outside the certified configuration (openai/gpt-6.1-sol at medium beside an anthropic lane)'); + expect(over.res).toContain('NEXT: judge the voices'); + expect(over.models['ship x17p1 outside-voice']).toEqual(high); + const named = await voicesRun(dir, env, 'named', { 'outside-voice': certified }); + expect(over.shas).toEqual(named.shas); + for (const t of Object.keys(named.models)) if (t !== 'ship x17p1 outside-voice') expect(over.models[t]).toEqual(named.models[t]); + }, 300_000); + test('an audit reader marked first settles before any specialist starts; its output is returned, never merged', async () => { const { dir, env } = repo(); const out = path.join(dir, '.state/run/p1'); diff --git a/test/ship-driver.test.ts b/test/ship-driver.test.ts index 9ef64eb8a..44e948c0e 100644 --- a/test/ship-driver.test.ts +++ b/test/ship-driver.test.ts @@ -210,7 +210,7 @@ describe('variant G: the call plan, observe budget, waits and audit readers (GST // Value: protects=audits that cannot block (Steps 7, 8, 9.2.1, 14.5) run on a small model at low effort with their sections' own briefs, and skip their reader when --audit decides them; fails_when=an audit reader inherits the lane's model and effort (#36: the coverage audit alone was 2.45M at Opus), or an audit brief is paraphrased; // why_new=audit model routing is new; seam=none; no_claim=the audits' recall test('audit readers run on a small model at low effort, named for Capy, and Phase 3 dispatches only what READER names', () => { - expect(flat(doc)).toContain('Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a small model at low effort; their findings go to the PR body'); + expect(flat(doc)).toContain('Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs at low effort on a small model, Step 14.5 once on Sonnet 5.5; findings go to the PR body'); expect(flat(doc)).toContain('`Coverage:` and Step 8\'s line come from `--audit` when no reader ran'); const capy = read('capy/skill/gstack/SKILL.md'); expect(capy).toContain('model: { model_id: "anthropic/claude-sonnet-5-5", reasoning_mode: "low" }');