mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test(qa-functional): fix mode requires only the happy scenario from the model (carried byte-identical from #3002 183b01f4..3e6074b4)
verifyQANativeRegression already reruns all eight webhook scenarios on the repaired source, so the model-side eight-scenario requirement in fix mode duplicated harness coverage and pushed qa-functional-webhook-fix past its budget. qa-only still requires every scenario.
This commit is contained in:
1 parent
1643cd94de
commit
271c14078b
4 files changed
+37
-13
No files matched your search
@@ -42,7 +42,7 @@ ${qaFixtureActor(entry.mode)}${entry.mode === 'qa' ? `
|
||||
|
||||
This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification. Complete these stages in order:
|
||||
1. Prove the regression red with a new native test under test/; existing tests remain read-only. Freeze all test files after red, repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts, and prove the unchanged regression green.
|
||||
2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe.${entry.family === 'webhook' ? ' Complete required webhook coverage: `happy`, `reject`, `duplicate`, `partial`, `concurrent-ab`, `concurrent-ba`, `cancel`, `dependency`. Run every still-unobserved scenario and recheck earlier scenarios affected by the repair. None of these scenarios is optional exploration. All eight scenarios are required coverage; a replay does not replace another scenario.' : ''} Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass.
|
||||
2. On the repaired source, run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe. Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass.
|
||||
3. Save the evidence and Markdown artifacts, then return their paths and the actual completion status.
|
||||
|
||||
The completion reserve is for both required verification and artifacts, not a signal to stop stage 2: the completion reserve does not end required coverage. Stop only exploration beyond the required contracts to protect that work. If required verification remains unfinished at the hard deadline, report incomplete; do not call it complete with a caveat.` : ''}
|
||||
|
||||
@@ -126,7 +126,7 @@ export function qaFunctionalVerdict(fixture: QAFunctionalFixture, mode: QAMode,
|
||||
const cancellations = probes.filter(probe => fixture.family === 'cli' ? nativeCommand(probe) === 'bun cancel.ts' : probe.observed.scenario === 'cancel');
|
||||
if (!cancellations.length) failures.push('missing cancellation observation');
|
||||
if (fixture.family === 'webhook') {
|
||||
for (const scenario of ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba']) {
|
||||
for (const scenario of mode === 'qa-only' ? ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba'] : ['happy']) {
|
||||
if (!probes.some(probe => probe.observed.scenario === scenario)) failures.push(`missing native ${scenario} probe`);
|
||||
}
|
||||
} else if (!probes.some(probe => probe.observed.args?.[0] === 'apply' && qaProbeClassification(probe.observed) === 'pass')) failures.push('missing adjacent valid CLI apply');
|
||||
|
||||
@@ -188,6 +188,22 @@ describe('functional evidence and native regression controls', () => {
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('webhook scenario coverage binds the report-only run; the fix run needs the happy path and relies on the post-repair recheck of all eight', () => {
|
||||
const fixture = createQAFunctionalFixture('webhook');
|
||||
try {
|
||||
const calls: NativeCall[] = [{ tool: 'Read', input: { file_path: 'qa/sections/system-functional.md' }, output: 'Functional QA native instruction read '.repeat(8) }];
|
||||
for (const scenario of ['happy', 'concurrent-ab', 'concurrent-ab', 'cancel', 'dependency']) recordedProbe(fixture, calls, ['probe.ts', scenario]);
|
||||
const result = nativeCapture(calls);
|
||||
const section = { path: 'qa/sections/system-functional.md', content: calls[0]!.output };
|
||||
const missing = (mode: 'qa' | 'qa-only') => qaFunctionalVerdict(fixture, mode, result, observation, {}, section)
|
||||
.filter(failure => /^missing native [a-z-]+ probe$/.test(failure)).sort();
|
||||
expect(missing('qa-only')).toEqual(['missing native duplicate probe', 'missing native partial probe', 'missing native reject probe', 'missing native concurrent-ba probe'].sort());
|
||||
expect(missing('qa')).toEqual([]);
|
||||
const noHappy = nativeCapture(calls.filter(call => !(call.tool === 'Bash' && call.input.command === 'bun run probe -- happy')));
|
||||
expect(qaFunctionalVerdict(fixture, 'qa', noHappy, observation, {}, section)).toContain('missing native happy probe');
|
||||
} finally { fixture.cleanup(); }
|
||||
});
|
||||
|
||||
test('native-shaped adverse receipts cannot hide stream, input, state or interruption failures', () => {
|
||||
const cli = createQAFunctionalFixture('cli', { healthy: true });
|
||||
const webhook = createQAFunctionalFixture('webhook', { healthy: true });
|
||||
|
||||
@@ -171,9 +171,9 @@ test('the native launcher consumes the family-specific actor boundary', () => {
|
||||
? 'The generic wrapper does NOT support wait'
|
||||
: 'bun cancel.ts is a CLI-only entrypoint, not part of this fixture');
|
||||
expect(prompt).not.toContain('parseInt');
|
||||
if (entry.family === 'webhook') {
|
||||
if (entry.family === 'webhook') expect(prompt).toContain('Choose their order from observations after the happy path');
|
||||
if (entry.family === 'webhook' && entry.mode === 'qa-only') {
|
||||
expect(prompt).toContain('All eight scenarios are required coverage; a replay does not replace another scenario');
|
||||
expect(prompt).toContain('Choose their order from observations after the happy path');
|
||||
} else {
|
||||
expect(prompt).not.toContain('All eight scenarios');
|
||||
}
|
||||
@@ -248,25 +248,33 @@ test('fix completion budgets for required repair and avoids duplicating preserve
|
||||
}
|
||||
});
|
||||
|
||||
test('webhook fix stage retains the required scenarios omitted by both R29 captures', () => {
|
||||
test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions stay bound by the report-only case', () => {
|
||||
// Both R29 fix-run captures completed the fix loop (happy path, replayed defect,
|
||||
// cancellation, dependency) and omitted only exploration scenarios. Eight-scenario
|
||||
// coverage is the report-only webhook case's contract; the fix case's harness
|
||||
// reruns all eight on the repaired source (verifyQANativeRegression).
|
||||
const captured = [
|
||||
{ id: 'ecd6da06-abd0-4299-8c33-e1b99a672325', scenarios: ['happy', 'partial', 'partial', 'concurrent-ab', 'partial', 'cancel', 'dependency', 'happy'], missing: ['reject', 'duplicate', 'concurrent-ba'] },
|
||||
{ id: '45722f13-a72c-4c01-87cc-8e17285ef8c4', scenarios: ['happy', 'concurrent-ab', 'concurrent-ab', 'concurrent-ab', 'happy', 'cancel', 'dependency'], missing: ['reject', 'duplicate', 'partial', 'concurrent-ba'] },
|
||||
];
|
||||
const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
const verification = prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence'));
|
||||
const required = ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'cancel', 'dependency'];
|
||||
for (const attempt of captured) {
|
||||
expect(required.filter(scenario => !attempt.scenarios.includes(scenario))).toEqual(attempt.missing);
|
||||
for (const scenario of required) expect(verification).toContain(`\`${scenario}\``);
|
||||
for (const scenario of ['happy', 'cancel', 'dependency']) expect(attempt.scenarios).toContain(scenario);
|
||||
expect(attempt.scenarios.some((scenario, index) => attempt.scenarios.indexOf(scenario) !== index && scenario !== 'happy')).toBe(true);
|
||||
}
|
||||
expect(verification).toContain('every still-unobserved scenario');
|
||||
expect(verification).toContain('recheck earlier scenarios affected by the repair');
|
||||
expect(verification).toContain('None of these scenarios is optional exploration');
|
||||
expect(prompt).toContain('the completion reserve does not end required coverage');
|
||||
expect(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })).not.toContain('`concurrent-ba`');
|
||||
const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
const verification = fix.slice(fix.indexOf('2. On the repaired source'), fix.indexOf('3. Save the evidence'));
|
||||
expect(verification).toContain('run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe');
|
||||
expect(verification).not.toContain('All eight scenarios');
|
||||
expect(fix).toContain('the completion reserve does not end required coverage');
|
||||
expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })));
|
||||
const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' });
|
||||
expect(report).toContain('All eight scenarios are required coverage; a replay does not replace another scenario');
|
||||
for (const scenario of required) expect(report).toContain(scenario);
|
||||
});
|
||||
|
||||
|
||||
test('fix-stage checkpoint provenance survives intervening native regression tests', () => {
|
||||
const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
expect(prompt).toContain('most recent completed native probe');
|
||||
|
||||
Reference in new issue
Block a user