From 3392d3f5d56e5605675cbfa53bf9694dddda44fd Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 17 Oct 2026 19:22:57 +0000 Subject: [PATCH] test(qa-functional): webhook fix runs re-verify the repaired invariant and rerun superseded captures (W3-6, W3-14) W3-6: stage 2 of the fix-run prompt asked only for the original, happy, cancel and dependency probes while verifyQANativeRegression reruns all eight, so a status-only repair that left partial broken reached the verifier (paid run 38986120559). The webhook fix prompt now states the capture budget up front (at most 5 pre-repair captures, then the 7 post-repair scenarios: duplicate, partial, concurrent-ab, concurrent-ba, happy, cancel, dependency; 12 in total; report incomplete if 5 cannot establish the defect). That list feeds the capture nudge (GSTACK_QA_REQUIRED_PROBES); QA_WEBHOOK_REQUIRED_SCENARIOS and the verdict are unchanged. W3-14: the fix prompt now says to rerun every pre-repair capture the verdict relies on after the final repair, and that a superseded row stays open until its command reran on the repaired source (paid run 39053431488 labelled a pre-repair duplicate superseded and never reran it). The detector and its negative test are unchanged; the captured shape joins them as a negative control. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/helpers/qa-functional-eval.ts | 15 +++- test/qa-functional-evidence.test.ts | 111 ++++++++++++++++++++++++++++ test/qa-functional-prompt.test.ts | 50 ++++++++++++- 3 files changed, 170 insertions(+), 6 deletions(-) diff --git a/test/helpers/qa-functional-eval.ts b/test/helpers/qa-functional-eval.ts index 947ab2f14..2bbe194b7 100644 --- a/test/helpers/qa-functional-eval.ts +++ b/test/helpers/qa-functional-eval.ts @@ -21,6 +21,15 @@ export const QA_FUNCTIONAL_CASES = [ { id: 'qa-functional-webhook-fix', family: 'webhook', mode: 'qa' }, ] as const; +/** Webhook fix runs: after the final source edit, every scenario of the repaired one-effect invariant, then happy, cancel and dependency. Feeds the capture nudge; the verdict keeps QA_WEBHOOK_REQUIRED_SCENARIOS. */ +export const QA_WEBHOOK_POST_REPAIR_SCENARIOS = ['duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'happy', 'cancel', 'dependency'] as const; + +/** Native probe commands the capture helper reports as not yet run (GSTACK_QA_REQUIRED_PROBES); informational, never a verdict. */ +export function qaFunctionalRequiredProbes(entry: { family: QAFamily; mode: QAMode }): string[] | undefined { + if (entry.family !== 'webhook') return; + return (entry.mode === 'qa' ? QA_WEBHOOK_POST_REPAIR_SCENARIOS : QA_WEBHOOK_REQUIRED_SCENARIOS[entry.mode]).map(scenario => `bun run probe -- ${scenario}`); +} + export function qaFunctionalPrompt(entry: { family: QAFamily; mode: QAMode }): string { return `Read ${entry.mode}/SKILL.md, qa/sections/scope.md, ${entry.mode}/sections/exploratory.md and qa/sections/system-functional.md in full to load gstack's /${entry.mode} functional workflow for this repository with ${entry.mode === 'qa' ? 'Full exploration and the Standard fix tier' : 'Full report-only exploration'}. All four reads are required before probing in this fixture, even when its surfaces and isolation are already established. This is a CLI/webhook product, not a browser or DX request. The generated local skill copies are the current evaluation inputs; all their installed-root references resolve inside this fixture. The common preamble alone was removed using the repository extractor. @@ -28,9 +37,9 @@ The fixture README and AGENTS define intended contracts, native tests and owned ${qaFixtureActor(entry.mode)}${entry.mode === 'qa' ? ` -This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification. Complete these stages in order: +This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification.${entry.family === 'webhook' ? ` Capture budget: at most 5 captures before the repair, then, after the final source edit, the ${QA_WEBHOOK_POST_REPAIR_SCENARIOS.length} scenarios of stage 2; ${5 + QA_WEBHOOK_POST_REPAIR_SCENARIOS.length} in total. If 5 pre-repair captures cannot establish and replay the defect, report incomplete.` : ''} Complete these stages in order: 1. Prove the regression red with a new native test under test/; existing tests remain read-only. Freeze all test files after red, repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts, and prove the unchanged regression green. -2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe. Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass. +2. On the repaired source, run ${entry.family === 'webhook' ? `the original failing probe and every other scenario of the repaired invariant (${QA_WEBHOOK_POST_REPAIR_SCENARIOS.slice(0, 4).join(', ')}), then the adjacent happy-path probe, cancellation and the unavailable-dependency probe` : 'the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe'}. After your final repair, rerun every pre-repair capture your verdict relies on; a superseded row stays open until its command reran on the repaired source. Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass. 3. Save the evidence and Markdown artifacts, then return their paths and the actual completion status. The completion reserve is for both required verification and artifacts, not a signal to stop stage 2: the completion reserve does not end required coverage. Stop only exploration beyond the required contracts to protect that work. If required verification remains unfinished at the hard deadline, report incomplete; do not call it complete with a caveat.` : ''} @@ -105,7 +114,7 @@ export async function runQAFunctionalCase(entry: { id: string; family: QAFamily; timeout, completionReserveMs: timeout / 4, testName: entry.id, runId, signal, env: { CLAUDE_CONFIG_DIR: fixture.config, GIT_OPTIONAL_LOCKS: '0', QA_STATE_ROOT: path.join(fixture.root, '.qa-state'), - ...(entry.family === 'webhook' ? { GSTACK_QA_REQUIRED_PROBES: JSON.stringify(QA_WEBHOOK_REQUIRED_SCENARIOS[entry.mode].map(scenario => `bun run probe -- ${scenario}`)) } : {}) }, + ...(qaFunctionalRequiredProbes(entry) ? { GSTACK_QA_REQUIRED_PROBES: JSON.stringify(qaFunctionalRequiredProbes(entry)) } : {}) }, }); return result; }, diff --git a/test/qa-functional-evidence.test.ts b/test/qa-functional-evidence.test.ts index 1d3dcb72d..8f1d36e0f 100644 --- a/test/qa-functional-evidence.test.ts +++ b/test/qa-functional-evidence.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect } from 'bun:test'; +import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; @@ -7,6 +8,7 @@ import { qaFunctionalVerdict, qaNativeProbes, qaProbeClassification, verifyQANat import { parseNDJSON, type SkillTestResult } from './helpers/session-runner'; import type { QAWriteObservation } from './helpers/qa-functional-observer'; import { observeQAWrites } from './helpers/qa-functional-observer'; +import { qaFunctionalRequiredProbes, QA_WEBHOOK_POST_REPAIR_SCENARIOS } from './helpers/qa-functional-eval'; const observation: QAWriteObservation = { complete: true, failures: [], events: [], changed: [], before: {}, after: {}, limits: ['representative event control; no live agent'] }; @@ -305,6 +307,115 @@ describe('functional evidence and native regression controls', () => { }); } + // Paid run 38986120559 (qa-functional-webhook-fix): the actor's status-only post-gate recheck and its concurrency regression. + const capturedStatusRepair = [" const state = readState();\n state.effects.push(", " const state = readState();\n if (state.jobs[id]?.status === 'complete') return;\n state.effects.push("] as const; + const capturedConcurrencyRegression = `import { test, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { processJob } from '../src/worker'; +import { readState, writeState } from '../src/storage'; + +test('concurrent workers that both read the pending job apply exactly one effect', async () => { + process.env.QA_STATE_ROOT = fs.mkdtempSync(path.join(path.resolve('.qa-state'), 'regression-1-')); + writeState({ jobs: { delivery: { cents: 7, status: 'pending', attempts: 0 } }, effects: [] }); + const releases: Record void> = {}; + const arrivals: Record> = {}; + const barrier = (label: string) => { + let arrived!: () => void; + arrivals[label] = new Promise(resolve => { arrived = resolve; }); + return () => { arrived(); return new Promise(resolve => { releases[label] = resolve; }); }; + }; + const a = processJob('delivery', { gate: barrier('a') }); + const b = processJob('delivery', { gate: barrier('b') }); + await Promise.all([arrivals.a, arrivals.b]); + releases.a(); await a; + releases.b(); await b; + const state = readState(); + expect(state.effects).toEqual([{ id: 'delivery', cents: 7 }]); + expect(state.jobs.delivery.status).toBe('complete'); +}); +`; + + test('captured 38986120559 status-only repair fails the independent recheck on partial; an effect-based repair passes', () => { + for (const repair of ['status', 'effect'] as const) { + const fixture = createQAFunctionalFixture('webhook'); + try { + fs.writeFileSync(path.join(fixture.root, 'test/worker.regression-1.test.ts'), capturedConcurrencyRegression); + const worker = path.join(fixture.root, 'src/worker.ts'); + const source = fs.readFileSync(worker, 'utf8'); + expect(source).toContain(capturedStatusRepair[0]); + fs.writeFileSync(worker, source.replace(capturedStatusRepair[0], repair === 'status' ? capturedStatusRepair[1] + : " const state = readState();\n if (!state.effects.some(effect => effect.id === id)) state.effects.push(")); + if (repair === 'status') { + expect(qaProbeClassification(JSON.parse(fixtureCommand(fixture.root, ['probe.ts', 'partial']).stdout))).toBe('product-defect'); + expect(() => verifyQANativeRegression(fixture)).toThrow('Candidate repair fails original or adjacent contract'); + } else { + const result = verifyQANativeRegression(fixture); + expect(result.red.exit).not.toBe(0); + expect(result.rechecks.map(probe => qaProbeClassification(probe))).toEqual(['pass', 'pass', 'pass', 'pass', 'pass', 'pass', 'pass', 'setup-blocked']); + } + } finally { fixture.cleanup(); } + } + }, 60_000); + + test('the capture nudge lists post-repair scenarios a pre-repair-only set never ran, and clears on the complete set', () => { + const fixture = createQAFunctionalFixture('webhook'); + try { + const required = qaFunctionalRequiredProbes({ family: 'webhook', mode: 'qa' })!; + let previous: string | undefined; + const capture = (scenario: string) => { + const id = String(Number(previous ?? '0') + 1).padStart(3, '0'); + const after = previous ? ['--after', previous, '--hypothesis', 'The previous scenario settled, so the next one checks another branch of the one-effect invariant.'] : []; + const run = spawnSync(process.execPath, ['bin/gstack-qa-evidence', 'capture', 'qa-reports', id, '--public', '--timeout-ms', '10000', ...after, '--', 'bun', 'run', 'probe', '--', scenario], { + cwd: fixture.root, encoding: 'utf8', timeout: 30_000, + env: { ...process.env, QA_STATE_ROOT: path.join(fixture.root, '.qa-state'), GIT_OPTIONAL_LOCKS: '0', GSTACK_QA_REQUIRED_PROBES: JSON.stringify(required) }, + }); + const line = run.stdout.split('\n').find(text => text.startsWith('QA_EVIDENCE ')); + expect(line, run.stderr).toBeDefined(); + const receipt = JSON.parse(line!.slice('QA_EVIDENCE '.length)); + expect(receipt.status, run.stderr).toBe('complete'); + previous = id; + return receipt.requiredRemaining as string[]; + }; + let remaining: string[] = []; + for (const scenario of ['happy', 'concurrent-ab', 'concurrent-ab', 'concurrent-ba']) remaining = capture(scenario); + expect(remaining).toEqual(['duplicate', 'partial', 'cancel', 'dependency'].map(scenario => `bun run probe -- ${scenario}`)); + for (const scenario of QA_WEBHOOK_POST_REPAIR_SCENARIOS) remaining = capture(scenario); + expect(remaining).toEqual([]); + } finally { fixture.cleanup(); } + }, 120_000); + + test('captured 39053431488 pre-repair duplicate labelled superseded without a rerun stays a failure (negative control)', () => { + // Capture order from #3252's webhook-fix run: happy, reject, duplicate, concurrent-ab x2, partial, the src/worker.ts + // repair, then concurrent-ab, partial, happy, cancel, dependency, concurrent-ba. Duplicate was never rerun. + const fixture = createQAFunctionalFixture('webhook'); + const healthy = createQAFunctionalFixture('webhook', { healthy: true }); + try { + const section = { path: 'qa/sections/system-functional.md', content: 'Functional QA native instruction read '.repeat(8) }; + const calls: NativeCall[] = [{ tool: 'Read', input: { file_path: section.path }, output: section.content }]; + const probe = (scenario: string) => recordedProbe(fixture, calls, ['probe.ts', scenario]); + for (const scenario of ['happy', 'reject', 'duplicate', 'concurrent-ab', 'concurrent-ab', 'partial']) probe(scenario); + const worker = path.join(fixture.root, 'src/worker.ts'); + const repaired = fs.readFileSync(path.join(healthy.root, 'src/worker.ts'), 'utf8'); + fs.writeFileSync(worker, repaired); + calls.push({ tool: 'Write', input: { file_path: worker, content: repaired }, output: 'File written.' }); + for (const scenario of ['concurrent-ab', 'partial', 'happy', 'cancel', 'dependency', 'concurrent-ba']) probe(scenario); + const evidenceFailures = (captured: NativeCall[], label: (scenario: string, index: number) => string | undefined) => { + const result = nativeCapture(captured); + const probes = qaNativeProbes(result); + const report = { revision: fixture.revision, cwd: fixture.root, runtime: `bun ${Bun.version}`, limits: ['Optional exporter is unavailable.'], + evidence: probes.map((row, index) => ({ command: row.command, contract: 'README.md', expected: 'One durable effect per delivery.', + classification: label(row.observed.scenario, index) ?? qaProbeClassification(row.observed), observed: row.observed })) }; + return qaFunctionalVerdict(fixture, 'qa', result, observation, report, section).filter(failure => failure.startsWith('missing exact sanitized evidence')); + }; + const duplicate = 'bun run probe -- duplicate'; + expect(evidenceFailures(calls, scenario => scenario === 'duplicate' ? 'superseded' : undefined)).toEqual([`missing exact sanitized evidence for ${duplicate}`]); + expect(evidenceFailures(calls, () => undefined)).toEqual([]); + probe('duplicate'); + expect(evidenceFailures(calls, (scenario, index) => scenario === 'duplicate' && index === 2 ? 'superseded' : undefined)).toEqual([]); + } finally { fixture.cleanup(); healthy.cleanup(); } + }, 60_000); + test('private evidence survives fixture cleanup and preserves sanitized native records', () => { const fixture = createQAFunctionalFixture('cli'); const artifacts = fs.mkdtempSync(path.join(os.tmpdir(), 'qa-evidence-')); diff --git a/test/qa-functional-prompt.test.ts b/test/qa-functional-prompt.test.ts index d7fef818d..07b118ed2 100644 --- a/test/qa-functional-prompt.test.ts +++ b/test/qa-functional-prompt.test.ts @@ -1,11 +1,11 @@ import { expect, test } from 'bun:test'; -import { qaFunctionalPrompt, QA_FUNCTIONAL_CASES } from './helpers/qa-functional-eval'; +import { qaFunctionalPrompt, qaFunctionalRequiredProbes, QA_FUNCTIONAL_CASES, QA_WEBHOOK_POST_REPAIR_SCENARIOS } from './helpers/qa-functional-eval'; import { qaCommandAllowed } from './helpers/qa-functional-observer'; import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { basename, dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { parseNDJSON } from './helpers/session-runner'; -import { qaFunctionalVerdict, qaNativeProbes } from './helpers/qa-functional-evidence'; +import { QA_WEBHOOK_REQUIRED_SCENARIOS, qaFunctionalVerdict, qaNativeProbes } from './helpers/qa-functional-evidence'; import { createQAFunctionalFixture, ownedPath } from './helpers/qa-functional-fixture'; import { validateQACheckpoints } from './helpers/qa-checkpoint-evidence'; import { computePaidCaseSelection } from '../scripts/test-paid-shards'; @@ -247,13 +247,57 @@ test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions for (const probe of ['original failing probe', 'happy-path probe', 'cancellation', 'unavailable-dependency probe']) expect(verification).toContain(probe); expect(verification).not.toContain('All eight scenarios'); expect(fix).toMatch(/completion reserve does not end required coverage/i); - expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' }))); + // Paid run 38986120559: a status-only repair fixed both concurrency orders but not partial, which stage 2 never asked for + // while verifyQANativeRegression reruns all eight. The webhook stage now names the repaired invariant's scenarios; the CLI + // stage keeps its four probes. + const cli = ((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })); + expect(cli).toContain('2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe.'); + expect(verification).toContain('2. On the repaired source, run the original failing probe and every other scenario of the repaired invariant (duplicate, partial, concurrent-ab, concurrent-ba), then the adjacent happy-path probe, cancellation and the unavailable-dependency probe.'); + expect(verification.slice(verification.indexOf('dependency probe. '))).toBe(cli.slice(cli.indexOf('dependency probe. '))); const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' }); expect(report).toMatch(/all eight scenarios are required coverage/i); for (const scenario of required) expect(report).toContain(scenario); }); +test('webhook fix budget names the post-repair scenarios up front, and they feed only the capture nudge', () => { + const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); + const budget = fix.slice(fix.indexOf('Capture budget:'), fix.indexOf('Complete these stages in order')); + expect(budget).toBe('Capture budget: at most 5 captures before the repair, then, after the final source edit, the 7 scenarios of stage 2; 12 in total. If 5 pre-repair captures cannot establish and replay the defect, report incomplete. '); + expect(fix.indexOf('Capture budget:')).toBeLessThan(fix.indexOf('1. Prove the regression red')); + for (const entry of QA_FUNCTIONAL_CASES.filter(entry => entry.id !== 'qa-functional-webhook-fix')) expect(qaFunctionalPrompt(entry)).not.toContain('Capture budget:'); + expect([...QA_WEBHOOK_POST_REPAIR_SCENARIOS]).toEqual(['duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'happy', 'cancel', 'dependency']); + expect(qaFunctionalRequiredProbes({ family: 'webhook', mode: 'qa' })).toEqual(QA_WEBHOOK_POST_REPAIR_SCENARIOS.map(scenario => `bun run probe -- ${scenario}`)); + expect(qaFunctionalRequiredProbes({ family: 'webhook', mode: 'qa-only' })).toEqual(QA_WEBHOOK_REQUIRED_SCENARIOS['qa-only'].map(scenario => `bun run probe -- ${scenario}`)); + expect(qaFunctionalRequiredProbes({ family: 'cli', mode: 'qa' })).toBeUndefined(); + expect(QA_WEBHOOK_REQUIRED_SCENARIOS.qa).toEqual(['happy', 'cancel', 'dependency']); + const source = readFileSync(join(import.meta.dir, 'helpers/qa-functional-eval.ts'), 'utf8'); + expect(source).toContain('GSTACK_QA_REQUIRED_PROBES: JSON.stringify(qaFunctionalRequiredProbes(entry))'); +}); + +test('fix runs rerun every pre-repair capture their verdict relies on; superseded rows stay open until rerun', () => { + // Paid run 39053431488 (#3252): capture 003 (duplicate) was labelled superseded and never rerun after the repair. + const rule = 'After your final repair, rerun every pre-repair capture your verdict relies on; a superseded row stays open until its command reran on the repaired source.'; + for (const entry of QA_FUNCTIONAL_CASES) { + const prompt = qaFunctionalPrompt(entry); + if (entry.mode === 'qa') expect(prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence'))).toContain(rule); + else expect(prompt).not.toContain(rule); + } +}); + +test('exploratory sections state one stop rule tied to per-rule input probes, and fix runs close superseded captures', () => { + const stop = 'Stop when scoped contracts are tested or blocked: each documented input rule needs its own probe breaking only that rule; rules left uncovered at a probe or time cap go in limits.'; + const fix = 'After a fix, every earlier capture is superseded, even from a client that prints no snapshot: rerun each one your verdict relies on, or report it as an open remainder.'; + for (const rel of ['qa/sections/exploratory.md', 'qa-only/sections/exploratory.md']) { + const section = readFileSync(join(import.meta.dir, '..', rel), 'utf8'); + expect(section.split(stop)).toHaveLength(2); + expect(section.match(/^Stop when\b.*$/gm)).toEqual([stop]); + expect(section.match(/input rule/g)).toHaveLength(1); + expect(section.includes(fix)).toBe(rel.startsWith('qa/')); + } + expect(readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8')).toContain('One probe per documented input rule'); +}); + test('fix-stage checkpoint provenance survives intervening native regression tests', () => { const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); expect(prompt).toMatch(/red\/green test output in the report, not in observed/i);