From 271c14078b73d0f58aea52ab5de8a95c53eb075b Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 18:34:36 +0000 Subject: [PATCH] test(qa-functional): fix mode requires only the happy scenario from the model (carried byte-identical from #3002 183b01f4..3e6074b4) verifyQANativeRegression already reruns all eight webhook scenarios on the repaired source, so the model-side eight-scenario requirement in fix mode duplicated harness coverage and pushed qa-functional-webhook-fix past its budget. qa-only still requires every scenario. --- test/helpers/qa-functional-eval.ts | 2 +- test/helpers/qa-functional-evidence.ts | 2 +- test/qa-functional-evidence.test.ts | 16 ++++++++++++++ test/qa-functional-prompt.test.ts | 30 ++++++++++++++++---------- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/test/helpers/qa-functional-eval.ts b/test/helpers/qa-functional-eval.ts index d21c83060..4e881c819 100644 --- a/test/helpers/qa-functional-eval.ts +++ b/test/helpers/qa-functional-eval.ts @@ -42,7 +42,7 @@ ${qaFixtureActor(entry.mode)}${entry.mode === 'qa' ? ` This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification. Complete these stages in order: 1. Prove the regression red with a new native test under test/; existing tests remain read-only. Freeze all test files after red, repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts, and prove the unchanged regression green. -2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe.${entry.family === 'webhook' ? ' Complete required webhook coverage: `happy`, `reject`, `duplicate`, `partial`, `concurrent-ab`, `concurrent-ba`, `cancel`, `dependency`. Run every still-unobserved scenario and recheck earlier scenarios affected by the repair. None of these scenarios is optional exploration. All eight scenarios are required coverage; a replay does not replace another scenario.' : ''} Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass. +2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe. Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass. 3. Save the evidence and Markdown artifacts, then return their paths and the actual completion status. The completion reserve is for both required verification and artifacts, not a signal to stop stage 2: the completion reserve does not end required coverage. Stop only exploration beyond the required contracts to protect that work. If required verification remains unfinished at the hard deadline, report incomplete; do not call it complete with a caveat.` : ''} diff --git a/test/helpers/qa-functional-evidence.ts b/test/helpers/qa-functional-evidence.ts index e34c9ee2a..4ecc095b3 100644 --- a/test/helpers/qa-functional-evidence.ts +++ b/test/helpers/qa-functional-evidence.ts @@ -126,7 +126,7 @@ export function qaFunctionalVerdict(fixture: QAFunctionalFixture, mode: QAMode, const cancellations = probes.filter(probe => fixture.family === 'cli' ? nativeCommand(probe) === 'bun cancel.ts' : probe.observed.scenario === 'cancel'); if (!cancellations.length) failures.push('missing cancellation observation'); if (fixture.family === 'webhook') { - for (const scenario of ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba']) { + for (const scenario of mode === 'qa-only' ? ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba'] : ['happy']) { if (!probes.some(probe => probe.observed.scenario === scenario)) failures.push(`missing native ${scenario} probe`); } } else if (!probes.some(probe => probe.observed.args?.[0] === 'apply' && qaProbeClassification(probe.observed) === 'pass')) failures.push('missing adjacent valid CLI apply'); diff --git a/test/qa-functional-evidence.test.ts b/test/qa-functional-evidence.test.ts index 675a5dac0..d4500cd16 100644 --- a/test/qa-functional-evidence.test.ts +++ b/test/qa-functional-evidence.test.ts @@ -188,6 +188,22 @@ describe('functional evidence and native regression controls', () => { } finally { fixture.cleanup(); } }); + test('webhook scenario coverage binds the report-only run; the fix run needs the happy path and relies on the post-repair recheck of all eight', () => { + const fixture = createQAFunctionalFixture('webhook'); + try { + const calls: NativeCall[] = [{ tool: 'Read', input: { file_path: 'qa/sections/system-functional.md' }, output: 'Functional QA native instruction read '.repeat(8) }]; + for (const scenario of ['happy', 'concurrent-ab', 'concurrent-ab', 'cancel', 'dependency']) recordedProbe(fixture, calls, ['probe.ts', scenario]); + const result = nativeCapture(calls); + const section = { path: 'qa/sections/system-functional.md', content: calls[0]!.output }; + const missing = (mode: 'qa' | 'qa-only') => qaFunctionalVerdict(fixture, mode, result, observation, {}, section) + .filter(failure => /^missing native [a-z-]+ probe$/.test(failure)).sort(); + expect(missing('qa-only')).toEqual(['missing native duplicate probe', 'missing native partial probe', 'missing native reject probe', 'missing native concurrent-ba probe'].sort()); + expect(missing('qa')).toEqual([]); + const noHappy = nativeCapture(calls.filter(call => !(call.tool === 'Bash' && call.input.command === 'bun run probe -- happy'))); + expect(qaFunctionalVerdict(fixture, 'qa', noHappy, observation, {}, section)).toContain('missing native happy probe'); + } finally { fixture.cleanup(); } + }); + test('native-shaped adverse receipts cannot hide stream, input, state or interruption failures', () => { const cli = createQAFunctionalFixture('cli', { healthy: true }); const webhook = createQAFunctionalFixture('webhook', { healthy: true }); diff --git a/test/qa-functional-prompt.test.ts b/test/qa-functional-prompt.test.ts index e59f26c67..109498d44 100644 --- a/test/qa-functional-prompt.test.ts +++ b/test/qa-functional-prompt.test.ts @@ -171,9 +171,9 @@ test('the native launcher consumes the family-specific actor boundary', () => { ? 'The generic wrapper does NOT support wait' : 'bun cancel.ts is a CLI-only entrypoint, not part of this fixture'); expect(prompt).not.toContain('parseInt'); - if (entry.family === 'webhook') { + if (entry.family === 'webhook') expect(prompt).toContain('Choose their order from observations after the happy path'); + if (entry.family === 'webhook' && entry.mode === 'qa-only') { expect(prompt).toContain('All eight scenarios are required coverage; a replay does not replace another scenario'); - expect(prompt).toContain('Choose their order from observations after the happy path'); } else { expect(prompt).not.toContain('All eight scenarios'); } @@ -248,25 +248,33 @@ test('fix completion budgets for required repair and avoids duplicating preserve } }); -test('webhook fix stage retains the required scenarios omitted by both R29 captures', () => { +test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions stay bound by the report-only case', () => { + // Both R29 fix-run captures completed the fix loop (happy path, replayed defect, + // cancellation, dependency) and omitted only exploration scenarios. Eight-scenario + // coverage is the report-only webhook case's contract; the fix case's harness + // reruns all eight on the repaired source (verifyQANativeRegression). const captured = [ { id: 'ecd6da06-abd0-4299-8c33-e1b99a672325', scenarios: ['happy', 'partial', 'partial', 'concurrent-ab', 'partial', 'cancel', 'dependency', 'happy'], missing: ['reject', 'duplicate', 'concurrent-ba'] }, { id: '45722f13-a72c-4c01-87cc-8e17285ef8c4', scenarios: ['happy', 'concurrent-ab', 'concurrent-ab', 'concurrent-ab', 'happy', 'cancel', 'dependency'], missing: ['reject', 'duplicate', 'partial', 'concurrent-ba'] }, ]; - const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); - const verification = prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')); const required = ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'cancel', 'dependency']; for (const attempt of captured) { expect(required.filter(scenario => !attempt.scenarios.includes(scenario))).toEqual(attempt.missing); - for (const scenario of required) expect(verification).toContain(`\`${scenario}\``); + for (const scenario of ['happy', 'cancel', 'dependency']) expect(attempt.scenarios).toContain(scenario); + expect(attempt.scenarios.some((scenario, index) => attempt.scenarios.indexOf(scenario) !== index && scenario !== 'happy')).toBe(true); } - expect(verification).toContain('every still-unobserved scenario'); - expect(verification).toContain('recheck earlier scenarios affected by the repair'); - expect(verification).toContain('None of these scenarios is optional exploration'); - expect(prompt).toContain('the completion reserve does not end required coverage'); - expect(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })).not.toContain('`concurrent-ba`'); + const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); + const verification = fix.slice(fix.indexOf('2. On the repaired source'), fix.indexOf('3. Save the evidence')); + expect(verification).toContain('run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe'); + expect(verification).not.toContain('All eight scenarios'); + expect(fix).toContain('the completion reserve does not end required coverage'); + expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' }))); + const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' }); + expect(report).toContain('All eight scenarios are required coverage; a replay does not replace another scenario'); + for (const scenario of required) expect(report).toContain(scenario); }); + test('fix-stage checkpoint provenance survives intervening native regression tests', () => { const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); expect(prompt).toContain('most recent completed native probe');