mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 15:04:37 +01:00
test(qa-functional): reserve webhook post-repair reruns first inside the unchanged 12-capture ceiling (W3-6/W3-14 confirm round)
The fix prompt now states precedence: post-repair reruns (the 7 stage-2 scenarios plus each distinct pre-repair command the verdict relies on, such as a reject probe) are reserved first, pre-repair exploration gets what remains, and reruns that cannot fit are reported as open. A free test fits a pre-repair reject plus the reproduced defect and its replay within 12 (11 captures). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
f16dce25f0
commit
4617b4ce98
2 files changed
+22
-3
No files matched your search
@@ -24,6 +24,9 @@ export const QA_FUNCTIONAL_CASES = [
|
||||
/** Webhook fix runs: after the final source edit, every scenario of the repaired one-effect invariant, then happy, cancel and dependency. Feeds the capture nudge; the verdict keeps QA_WEBHOOK_REQUIRED_SCENARIOS. */
|
||||
export const QA_WEBHOOK_POST_REPAIR_SCENARIOS = ['duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'happy', 'cancel', 'dependency'] as const;
|
||||
|
||||
/** Webhook fix capture ceiling, unchanged at 12: post-repair reruns are reserved first and pre-repair exploration gets the rest. */
|
||||
export const QA_WEBHOOK_FIX_CAPTURE_LIMIT = 12;
|
||||
|
||||
/** Native probe commands the capture helper reports as not yet run (GSTACK_QA_REQUIRED_PROBES); informational, never a verdict. */
|
||||
export function qaFunctionalRequiredProbes(entry: { family: QAFamily; mode: QAMode }): string[] | undefined {
|
||||
if (entry.family !== 'webhook') return;
|
||||
@@ -37,7 +40,7 @@ The fixture README and AGENTS define intended contracts, native tests and owned
|
||||
|
||||
${qaFixtureActor(entry.mode)}${entry.mode === 'qa' ? `
|
||||
|
||||
This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification.${entry.family === 'webhook' ? ` Capture budget: at most 5 captures before the repair, then, after the final source edit, the ${QA_WEBHOOK_POST_REPAIR_SCENARIOS.length} scenarios of stage 2; ${5 + QA_WEBHOOK_POST_REPAIR_SCENARIOS.length} in total. If 5 pre-repair captures cannot establish and replay the defect, report incomplete.` : ''} Complete these stages in order:
|
||||
This is a fix run, not an optional report-only handoff: a reproduced in-tier defect requires the authorized native regression, repair and verification.${entry.family === 'webhook' ? ` Capture budget: ${QA_WEBHOOK_FIX_CAPTURE_LIMIT} captures in total. Reserve the post-repair reruns first: the ${QA_WEBHOOK_POST_REPAIR_SCENARIOS.length} scenarios of stage 2 plus each distinct pre-repair command your verdict relies on that they do not repeat (for example a reject probe); pre-repair exploration gets only what remains. If those reruns cannot all fit, report the rest as open. If the remaining pre-repair captures cannot establish and replay the defect, report incomplete.` : ''} Complete these stages in order:
|
||||
1. Prove the regression red with a new native test under test/; existing tests remain read-only. Freeze all test files after red, repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts, and prove the unchanged regression green.
|
||||
2. On the repaired source, run ${entry.family === 'webhook' ? `the original failing probe and every other scenario of the repaired invariant (${QA_WEBHOOK_POST_REPAIR_SCENARIOS.slice(0, 4).join(', ')}), then the adjacent happy-path probe, cancellation and the unavailable-dependency probe` : 'the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe'}. After your final repair, rerun every pre-repair capture your verdict relies on; a superseded row stays open until its command reran on the repaired source. Preserve their actual JSON and checkpoints. A green test suite does not substitute for these native probes. Expected dependency blockage stays blocked, never pass.
|
||||
3. Save the evidence and Markdown artifacts, then return their paths and the actual completion status.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { qaFunctionalPrompt, qaFunctionalRequiredProbes, QA_FUNCTIONAL_CASES, QA_WEBHOOK_POST_REPAIR_SCENARIOS } from './helpers/qa-functional-eval';
|
||||
import { qaFunctionalPrompt, qaFunctionalRequiredProbes, QA_FUNCTIONAL_CASES, QA_WEBHOOK_FIX_CAPTURE_LIMIT, QA_WEBHOOK_POST_REPAIR_SCENARIOS } from './helpers/qa-functional-eval';
|
||||
import { qaCommandAllowed } from './helpers/qa-functional-observer';
|
||||
import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { basename, dirname, join } from 'node:path';
|
||||
@@ -263,7 +263,7 @@ test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions
|
||||
test('webhook fix budget names the post-repair scenarios up front, and they feed only the capture nudge', () => {
|
||||
const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
const budget = fix.slice(fix.indexOf('Capture budget:'), fix.indexOf('Complete these stages in order'));
|
||||
expect(budget).toBe('Capture budget: at most 5 captures before the repair, then, after the final source edit, the 7 scenarios of stage 2; 12 in total. If 5 pre-repair captures cannot establish and replay the defect, report incomplete. ');
|
||||
expect(budget).toBe('Capture budget: 12 captures in total. Reserve the post-repair reruns first: the 7 scenarios of stage 2 plus each distinct pre-repair command your verdict relies on that they do not repeat (for example a reject probe); pre-repair exploration gets only what remains. If those reruns cannot all fit, report the rest as open. If the remaining pre-repair captures cannot establish and replay the defect, report incomplete. ');
|
||||
expect(fix.indexOf('Capture budget:')).toBeLessThan(fix.indexOf('1. Prove the regression red'));
|
||||
for (const entry of QA_FUNCTIONAL_CASES.filter(entry => entry.id !== 'qa-functional-webhook-fix')) expect(qaFunctionalPrompt(entry)).not.toContain('Capture budget:');
|
||||
expect([...QA_WEBHOOK_POST_REPAIR_SCENARIOS]).toEqual(['duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'happy', 'cancel', 'dependency']);
|
||||
@@ -275,6 +275,22 @@ test('webhook fix budget names the post-repair scenarios up front, and they feed
|
||||
expect(source).toContain('GSTACK_QA_REQUIRED_PROBES: JSON.stringify(qaFunctionalRequiredProbes(entry))');
|
||||
});
|
||||
|
||||
test('webhook fix budget fits a pre-repair reject plus the reproduced defect inside the unchanged 12-capture ceiling', () => {
|
||||
expect(QA_WEBHOOK_FIX_CAPTURE_LIMIT).toBe(12);
|
||||
const probe = (scenario: string) => `bun run probe -- ${scenario}`;
|
||||
const preRepair = ['reject', 'duplicate', 'duplicate'].map(probe);
|
||||
const reliedOn = [...new Set(preRepair)];
|
||||
const postRepair = [...new Set([...QA_WEBHOOK_POST_REPAIR_SCENARIOS.map(probe), ...reliedOn])];
|
||||
expect(postRepair).toEqual([...QA_WEBHOOK_POST_REPAIR_SCENARIOS.map(probe), probe('reject')]);
|
||||
expect(preRepair.length + postRepair.length).toBe(11);
|
||||
expect(preRepair.length + postRepair.length).toBeLessThanOrEqual(QA_WEBHOOK_FIX_CAPTURE_LIMIT);
|
||||
expect(QA_WEBHOOK_FIX_CAPTURE_LIMIT - postRepair.length).toBeGreaterThanOrEqual(preRepair.length);
|
||||
const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
const budget = fix.slice(fix.indexOf('Capture budget:'), fix.indexOf('Complete these stages in order'));
|
||||
expect(budget.indexOf('Reserve the post-repair reruns first')).toBeLessThan(budget.indexOf('pre-repair exploration gets only what remains'));
|
||||
expect(budget).toContain('report the rest as open');
|
||||
});
|
||||
|
||||
test('fix runs rerun every pre-repair capture their verdict relies on; superseded rows stay open until rerun', () => {
|
||||
// Paid run 39053431488 (#3252): capture 003 (duplicate) was labelled superseded and never rerun after the repair.
|
||||
const rule = 'After your final repair, rerun every pre-repair capture your verdict relies on; a superseded row stays open until its command reran on the repaired source.';
|
||||
|
||||
Reference in new issue
Block a user