From 43f394671a9c6e8b8c1930cdad14a19bd1e16c90 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:12:22 +0000 Subject: [PATCH] test(W7c): qa-probe-gates keeps loop order, bounds and the CI-proven Read gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t07 F3, F4, F5). One test file. This file stays the single owner of shared QA loop ordering (qa-functional-prompt defers to it). Removed assertion groups (invariant -> remaining protection): - `> **STOP.** Before any probe ...` prefix pin (t07 F3, retire) -> parent QA loads methods before any probe: kept free meaning check on "before any probe, including plan checks, complete the ordered scope/method Reads" and its order before the required-checks step. - Preparation prose in the stage list ("in full and select the surfaces", "2. Read the selected surface methods below in full", "Write a **charter**", "Start once before baseline:") -> stage order scope -> methods -> charter -> deadline start -> loop: kept as structural anchors (section Reads, surface labels, `## 1. Charter and preflight`, `bun G start D SECONDS`, step 1 title); added a reorder control moving a method Read after the charter. - Bounds/layout sentences ("an unlimited mode uses the caller's bound", "announce finite command timeouts", "EARLIER_UTC = caller's absolute deadline, if set", "in the probe directory, beside its deadline if bounded", duplicate SECONDS pins in the mode-bounds test, "charters as Markdown in the report", the full "observed: its exact decoded child JSON ..." sentence) -> numeric bounds and field names are kept free checks; the observed-field shape is enforced by the native checkpoint validator (qa-functional-prompt validator tests, free) and gate E2Es qa-functional-cli-report, qa-functional-webhook-report (gate). - Publication prose ("No drafts/placeholders", "and replay limits in the report", "An absolute state path is not itself a secret", "If capture is incomplete, report that limit instead of reconstructing it") -> notes are immutable and unsafe values withheld: kept free checks on "corrections cannot repair published notes", never invent a substitute, withhold secrets/unsafe values and stop the chain; validator-backed in qa-functional-prompt; gate E2Es review-exploratory-small-cli, ship-exploratory-small-cli (gate). - Loop-step prose ("Guard if bounded; await completion", "Run that exact probe; G enforces ..." duplicate, "then minimize via those gates", "Expiry leaves confirmation/minimization incomplete", "for each affected revalidation", "If bounded, run", "Reuse Step 4's ... completed Reads", "Finish missing methods before charters") -> loop has 5 steps, guarded by G, replays via steps 2–3, revalidation returns to step 2: kept free checks on step count, `G enforces the deadline`, `bun G status D`, `via steps 2–3`, `return to step 2`, refusals as not-run, do not repeat completed Reads. - CI-fixture clause "description, section index or remembered method is not a completed instruction Read" -> covered by the kept CI-proven clauses (`## 0. Preparation gate`, await each Read, complete missing Reads before Charter) and gate E2E ship-exploratory-small-cli (gate; the CI case). Kept (meaning checks, case-insensitive where prose): CI-proven "await their results before the first probe" rule (comment + mutation kept), Reads in order, no repeated Reads, SECONDS=30/900, shorter mode/caller limit, no D without a total limit, finite timeouts, REPORT_DIR layout and caller paths win, plan checks after smoke with no smoke guard and no clock reset, caller cap, not-run on expiry, await clock results, never reset D/bypass G, invalid/missing D stops probes, expiry -> write the report, Step 4 read-only, isolation/permission preflight before setup, field names, QA_DEADLINE receipts, all semantic-flip mutation controls, method/report resource paths, helper commands. Cosmetic pins dropped: STOP / AND capitals, bold STOP prefix. Mutation log: .context/mutation-qa-probe-gates.md (22 mutations, 0 unexpected: 3 reorders, 4 markers, 8 safety red; 7 rewordings green). --- test/qa-probe-gates.test.ts | 140 ++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 80 deletions(-) diff --git a/test/qa-probe-gates.test.ts b/test/qa-probe-gates.test.ts index 362225875..59ece4a61 100644 --- a/test/qa-probe-gates.test.ts +++ b/test/qa-probe-gates.test.ts @@ -7,14 +7,12 @@ import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates'; import { HOST_PATHS } from '../scripts/resolvers/types'; function assertPreparation(text: string) { - expect(text).toContain('Complete these Reads in order before writing charters or probing'); - expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./); - expect(text).toContain('Do not repeat a Read already completed in this invocation'); - const stages = ['1. Read `sections/scope.md`', 'in full and select the surfaces', - '2. Read the selected surface methods below in full', '**Functional surfaces:**', - 'Read `sections/system-functional.md` in full.', '**Browser surfaces only:**', - 'Read `sections/qa-patterns.md` in full.', '## 1. Charter and preflight', - 'Write a **charter**', 'Start once before baseline:', '1. First demonstrate success']; + expect(text).toMatch(/reads in order before writing charters or probing/i); + expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./i); + expect(text).toMatch(/do not repeat a Read already completed/i); + const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`', + '**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight', + 'bun G start D SECONDS', '1. First demonstrate success']; const positions = stages.map(stage => text.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); @@ -24,36 +22,35 @@ function assertPreparation(text: string) { } function assertBoundsAndLayout(text: string) { - for (const contract of [ - 'Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900', - 'Functional Full, Quick and Regression have no default total timer', - "Set SECONDS to the shorter mode/caller limit", - "an unlimited mode uses the caller\'s bound", - 'Without a total time limit, do not start D', - 'announce finite command timeouts', - "EARLIER_UTC = caller\'s absolute deadline, if set", - 'Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.', - 'in the probe directory, beside its deadline if bounded', - ]) expect(text).toContain(contract); - expect(text.indexOf('Caller paths win.')).toBeLessThan(text.indexOf('Start once before baseline:')); + for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract); + for (const rule of [ + /functional full, quick and regression have no default total timer/i, + /shorter mode\/caller limit/i, + /without a total time limit, do not start D/i, + /finite command timeouts/i, + /mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i, + /one final report at REPORT_DIR/i, + /caller paths win/i, + ]) expect(text).toMatch(rule); + expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS')); } function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) { const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' '); - for (const contract of [ - 'Follow the shared Probe loop for smoke checks and replays until the smoke limit', - 'Then run required plan checks and revalidation, even after smoke expires', - 'using the same procedure but no smoke guard; never reset the clock', - "Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline", - 'When the caller\'s deadline expires, mark unfinished checks not-run', - 'Await clock/guard results before acting', - ]) expect(step).toContain(contract); + for (const rule of [ + /shared Probe loop for smoke checks and replays until the smoke limit/i, + /required plan checks and revalidation, even after smoke expires/i, + /same procedure but no smoke guard; never reset the clock/i, + /capped at the caller's remaining time/i, + /deadline expires, mark unfinished checks not-run/i, + /await clock\/guard results before acting/i, + ]) expect(step).toMatch(rule); expect(step.indexOf('Follow the shared Probe loop')).toBeLessThan(step.indexOf('Then run required plan checks')); expect(step.indexOf('Then run required plan checks')).toBeLessThan(step.indexOf('same procedure')); - for (const contract of ['First demonstrate success: output AND durable effects', - 'Wait for successful checkpoint publication before dispatch', - 'Replay the exact failing command/request from the same initial fixture state']) { - expect(shared).toContain(contract); + for (const rule of [/demonstrate success: output and durable effects/i, + /wait for successful checkpoint publication before dispatch/i, + /replay the exact failing command\/request from the same initial fixture state/i]) { + expect(shared).toMatch(rule); } } @@ -67,8 +64,7 @@ describe('QA probe entry and checkpoint gates', () => { expect(old).toContain('Read `sections/system-functional.md`'); const current = generateQAExploratory({ host: 'claude', skillName: 'qa-only', tmplPath: '', paths: HOST_PATHS.claude }); for (const clause of ['## 0. Preparation gate', 'Await each successful Read result before continuing', - 'description, section index or remembered method is not a completed instruction Read', - 'If either required Read is missing, complete it now before Charter and preflight']) { + 'complete it now before Charter and preflight']) { expect(old).not.toContain(clause); expect(current).toContain(clause); } @@ -88,13 +84,13 @@ describe('QA probe entry and checkpoint gates', () => { const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] }; const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' '); const body = generateQAReview(ctx).replace(/\s+/g, ' '); - expect(preflight).toContain('Resolve QA\'s `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory'); + expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i); expect(body).toContain('Read QA\'s `sections/browser-setup.md`'); expect(body).toContain('Read QA\'s `templates/functional-report-template.md`'); if (skillName === 'review') { expect(body).toContain('Read QA\'s `templates/qa-report-template.md`'); - expect(body).toContain('Reuse Step 4\'s surfaces and completed Reads'); - expect(body).toContain('Finish missing methods before charters; do not repeat completed Reads'); + expect(body).toContain('Step 4\'s surfaces'); + expect(body).toMatch(/do not repeat completed Reads/i); } } } @@ -110,19 +106,14 @@ describe('QA probe entry and checkpoint gates', () => { const positions = stages.map(stage => text.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(text).toContain('No drafts/placeholders'); - expect(text).toContain('corrections cannot repair published notes'); + expect(text).toMatch(/corrections cannot repair published notes/i); if (skillName === 'qa-only') { - expect(text).toContain('never invent a substitute path, identity or state'); - expect(text).toContain('For actual secrets/private payloads, withhold those values'); - expect(text).toContain('and replay limits in the report'); - expect(text).toContain('stop the affected probe chain'); - expect(text).toContain('An absolute state path is not itself a secret'); + expect(text).toMatch(/never invent a substitute path, identity or state/i); + expect(text).toMatch(/secrets\/private payloads, withhold those values/i); + expect(text).toMatch(/stop the affected probe chain/i); } else { - expect(text).toContain('No drafts/placeholders or invented safe-path redactions'); - expect(text).toContain('Withhold unsafe values, disclose limits and stop that chain'); + expect(text).toMatch(/withhold unsafe values, disclose limits and stop that chain/i); } - if (skillName === 'qa-only') expect(text).toContain('If capture is incomplete, report that limit instead of reconstructing it'); } } }); @@ -132,13 +123,7 @@ describe('QA probe entry and checkpoint gates', () => { const text = generateQAExploratory({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude }); for (const contract of [ 'Reuse resolved REPORT_DIR', - 'charters as Markdown in the report', - "Set SECONDS to the shorter mode/caller limit", - 'Without a total time limit, do not start D', - 'Browser Quick: SECONDS=30', - 'Browser Full/Regression: SECONDS=900', 'exactly four top-level fields:', 'observationCommand:', 'observed:', 'hypothesis:', 'nextCommand:', - 'observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text', 'QA_DEADLINE receipts are not observations', ]) expect(text).toContain(contract); assertBoundsAndLayout(text); @@ -159,17 +144,15 @@ describe('QA probe entry and checkpoint gates', () => { const write = text.indexOf('**Publish before probing.**'); expect(decision).toBeGreaterThan(-1); expect(decision).toBeLessThan(write); - expect(text.slice(decision, write)).toContain('write the report, not a checkpoint'); - expect(text.slice(decision, write)).toContain('If expired or no safe next probe remains'); + expect(text.slice(decision, write)).toMatch(/write the report, not a checkpoint/i); + expect(text.slice(decision, write)).toMatch(/if expired/i); expect(text.slice(decision, write)).not.toContain('If done or blocked'); if (skillName === 'qa-only') { - expect(text).toContain('Check fields before publication'); - expect(text).toContain('retain the entire result unchanged, including owned fixture paths, IDs, hashes'); + expect(text).toMatch(/retain the entire result unchanged/i); } else { - expect(text).toContain('Preserve every safe program-JSON key/value'); - expect(text).toContain('Preserve every safe program-JSON key/value and identity hash unchanged'); + expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i); } - expect(text).toContain('Q supplies observed; never transcribe it'); + expect(text).toMatch(/Q supplies observed; never transcribe it/i); } } }); @@ -182,20 +165,16 @@ describe('QA probe entry and checkpoint gates', () => { expect(text).toContain('/gstack-qa-deadline'); expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]'); expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`'); - expect(text).toContain('Never reset D/bypass G'); - expect(text).toContain('invalid/missing D stops probes'); - expect(steps[0]).toContain('demonstrate success: output AND durable effects'); - expect(steps[0]).toContain('Guard if bounded; await completion'); - expect(steps[1]).toContain('If bounded, run `bun G status D`'); + expect(text).toMatch(/never reset D\/bypass G/i); + expect(text).toMatch(/invalid\/missing D stops probes/i); + expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i); + expect(steps[1]).toContain('`bun G status D`'); expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**')); - expect(steps[1]).toContain('STOP exploration; write the report, not a checkpoint'); - expect(steps[2]).toContain('Run that exact probe; G enforces the deadline when bounded'); + expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i); expect(steps[2]).toContain('G enforces the deadline'); - expect(steps[2]).toContain('Report refusals as not-run'); + expect(steps[2]).toMatch(/refusals as not-run/i); expect(steps[3]).toContain('via steps 2–3'); - expect(steps[3]).toContain('then minimize via those gates'); - expect(steps[3]).toContain('Expiry leaves confirmation/minimization incomplete'); - expect(steps[4]).toContain('return to step 2 for each affected revalidation'); + expect(steps[4]).toContain('return to step 2'); } }); @@ -204,25 +183,25 @@ describe('QA probe entry and checkpoint gates', () => { for (const skillName of ['review', 'ship']) { const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] }; const text = (skillName === 'review' ? generateQAReviewPreflight(ctx) : '') + generateQAReview(ctx); - expect(text).toContain('> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below'); + const gate = text.search(/before any probe, including plan checks, complete the ordered scope\/method Reads/i); + expect(gate).toBeGreaterThan(-1); expect(text).toContain('{{QA_RESOURCE:exploratory}}'); expect(text).not.toContain('{{QA_RESOURCE:scope}}'); expect(text).not.toContain('Read `sections/system-functional.md`'); const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }); assertPreparation(shared); - expect(text).toContain('Before any probe, including plan checks'); assertPlanExecution(text); const required = text.indexOf(skillName === 'review' ? '**2. Check readiness and list required checks.**' : '**2. List required checks'); expect(required).toBeGreaterThan(-1); - expect(text.indexOf('> **STOP.**')).toBeLessThan(required); + expect(gate).toBeLessThan(required); if (skillName === 'review') { const isolation = text.indexOf('**1. Set the charter and isolation.**'); const setup = text.indexOf("Read QA's `sections/browser-setup.md`"); - expect(isolation).toBeGreaterThan(text.indexOf('> **STOP.**')); + expect(isolation).toBeGreaterThan(gate); expect(setup).toBeGreaterThan(required); - expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup'); - expect(text).toContain('Step 4 is read-only: defer charters, setup and probes to Step 4.7'); + expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i); + expect(text).toMatch(/Step 4 is read-only: defer charters, setup and probes to Step 4\.7/i); } else { const setup = text.indexOf("For browsers, Read QA's `sections/browser-setup.md`"); expect(setup).toBeGreaterThan(required); @@ -239,8 +218,9 @@ describe('QA probe entry and checkpoint gates', () => { for (const changed of [ method + '\n' + text.replace(method, ''), text.replace(method, method + '\n' + method), - 'Write a **charter**\n' + text, + '## 1. Charter and preflight\n' + text, text.replace('Do not repeat a Read already completed in this invocation', 'Repeat all Reads'), + text.replace('Read `sections/qa-patterns.md` in full.', '').replace('## 1. Charter and preflight', '## 1. Charter and preflight\nRead `sections/qa-patterns.md` in full.'), // ci-36641820398-1-gate-census-7 ship-exploratory-small-cli dispatched its first probe with the resource Reads. text.replace(' Await their results before the first probe, never in the same response.', ''), ]) expect(() => assertPreparation(changed)).toThrow(); @@ -303,8 +283,8 @@ describe('QA probe entry and checkpoint gates', () => { test('plan execution waits for the actual shared method reads, not just collection', () => { const text = generatePlanVerificationExec({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude }); - expect(text).toContain('Do not invoke an entire QA skill or start probes here'); + expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i); expect(text).toContain('Before the first plan command, complete Step 9.2.1'); - expect(text).toContain('method Reads and the shared probe loop'); + expect(text).toMatch(/method Reads and the shared probe loop/i); }); });