diff --git a/scripts/resolvers/qa.ts b/scripts/resolvers/qa.ts index e4a8fbe49..e223e1a70 100644 --- a/scripts/resolvers/qa.ts +++ b/scripts/resolvers/qa.ts @@ -76,30 +76,33 @@ For /qa and /qa-only: - Browser Quick: SECONDS=30. Browser Full/Regression: SECONDS=900. - Functional Full, Quick and Regression have no default total timer. Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound. -Without a total time limit, do not start D; announce finite command timeouts. +Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts. Stop when scoped contracts are tested or blocked. Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win. Names used below (quote every path): -- R: this surface's owned probe directory, chosen above. D: \`R/deadline.json\`, which exists only for a bounded run. +- PROBE_DIR: this surface's owned probe directory, chosen above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run. - SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one. -- G = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates D, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline. -- Q = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`R/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`. +- DEADLINE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates DEADLINE_FILE, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline. +- EVIDENCE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`PROBE_DIR/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`. -Start once before baseline: \`bun G start D SECONDS [EARLIER_UTC]\` if bounded. +Start once before baseline: \`bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]\` if bounded. EARLIER_UTC = caller's absolute deadline, if set. -Functional: \`bun Q capture R NNN [--public] --deadline D -- COMMAND ARGS\`. -Unbounded: use \`--timeout-ms MS\` instead of \`--deadline D\`. ---public requires approved public/synthetic output; Q screens credentials. For complete private captures, await a safe Read of \`R/.qa-evidence/NNN/observation.json\` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints. -Bounded browsers: \`bun G run D -- COMMAND ARGS\`. No detached probes. -Never reset D/bypass G. Expiry or invalid/missing D stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations. +Functional: \`bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS\`. +Unbounded: use \`--timeout-ms MS\` instead of \`--deadline DEADLINE_FILE\`. +--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of \`PROBE_DIR/.qa-evidence/NNN/observation.json\` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints. +Bounded browsers: \`bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS\`. No detached probes. +Never reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations. ## 2. Probe loop Each probe is one native command/interaction plus checks, excluding bookkeeping. Never batch probes. +How one probe works: (1) if bounded, check the clock and choose the next command; +(2) publish a checkpoint naming it (functional: \`capture --after\` publishes it); +(3) run exactly that command via DEADLINE_TOOL or EVIDENCE_TOOL, read its result, repeat. 1. First demonstrate success: output AND durable effects. Guard if bounded; await completion. -2. **Decide whether another probe is needed.** If bounded, run \`bun G status D\`. +2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`. If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint. ${reportOnly ? ` **Classify the last result before copying it.** For public or synthetic observations, retain the entire result unchanged, including owned fixture paths, IDs, hashes and @@ -119,11 +122,11 @@ ${reportOnly ? ` For guarded text, copy the complete span between the guard's Preserve every safe program-JSON key/value and identity hash unchanged. Withhold unsafe values, disclose limits and stop that chain. Check fields before publication. No drafts/placeholders or invented safe-path redactions; corrections cannot repair published notes. - Functional: do not write this file; the next capture publishes it: \`bun Q capture R NNN --deadline D --after PREV --hypothesis 'why' -- CMD\` (PREV: the last complete capture's ID). Q supplies observed; never transcribe it. + Functional: do not write this file; the next capture publishes it: \`bun EVIDENCE_TOOL capture PROBE_DIR NNN --deadline DEADLINE_FILE --after PREV --hypothesis 'why' -- CMD\` (PREV: the last complete capture's ID). EVIDENCE_TOOL supplies observed; never transcribe it. Browser checkpoints use Write: create the four-field file yourself. Wait for successful checkpoint publication before dispatch. Never backfill or overwrite notes. -3. Run that exact probe; G enforces the deadline when bounded. +3. Run that exact probe; DEADLINE_TOOL enforces the deadline when bounded. Report refusals as not-run; retain initial state/inputs/results. Repeat from step 2. 4. Replay the exact failing command/request from the same initial fixture state via steps 2–3 (same native command, fresh capture ID) ${reportOnly ? 'to confirm it' : 'before repair'}, then minimize via those gates. Expiry leaves confirmation/minimization incomplete. @@ -147,7 +150,7 @@ Never freeze buggy output, weaken tests or delete valid red tests.`} ## 4. Final report Use the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests. -Write R/annotations.json {evidence: [{capture, command, contract, expected, classification}], limits} (browser-only: evidence [], checkpoints in limits); before Markdown \`bun Q materialize R annotations.json\` (fills observed/metadata; prints reportLinks); you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. +Write PROBE_DIR/annotations.json {evidence: [{capture, command, contract, expected, classification}], limits} (browser-only: evidence [], checkpoints in limits); before Markdown \`bun EVIDENCE_TOOL materialize PROBE_DIR annotations.json\` (fills observed/metadata; prints reportLinks); you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. Evidence is invocation-local${reportOnly ? '.' : '; /ship reruns once per invocation.'} Missing prerequisites/expectations/observations, timeouts and refusal never pass. Pass requires all required current-input contracts to pass with no required remainder. @@ -260,7 +263,7 @@ Never install, import cookies or bootstrap tests. Functional-only skips browser **3. Run smoke and plan checks.** Follow the shared Probe loop for smoke checks and replays until the smoke limit. -Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside D; they skip \`G status D\` and use \`--timeout-ms\`, not \`--deadline D\`. Post-expiry smoke rechecks are not-run. +Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. Post-expiry smoke rechecks are not-run. Use finite command timeouts, capped at the caller's remaining time if it has a deadline.${ship ? '' : ' /review sets none; only an invoker-supplied EARLIER_UTC counts.'} Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run. diff --git a/test/helpers/qa-probe-names.ts b/test/helpers/qa-probe-names.ts new file mode 100644 index 000000000..1e203dacc --- /dev/null +++ b/test/helpers/qa-probe-names.ts @@ -0,0 +1,31 @@ +/** + * Structural reader for the shared QA exploratory "Names used below" list. + * Tests check the commands built from these names (argument order, the CLI each + * name is bound to) instead of pinning the names' spelling. + */ +export interface QaProbeNames { + dir: string; + deadline: string; + guard: string; + recorder: string; +} + +export function qaProbeNames(text: string): QaProbeNames { + const start = text.indexOf('Names used below'); + if (start < 0) throw new Error('QA exploratory text has no "Names used below" list'); + const block = text.slice(start, text.indexOf('\n\n', start)); + const bound = (pattern: RegExp, what: string) => { + const match = block.match(pattern); + if (!match) throw new Error(`QA probe names: no ${what} binding in:\n${block}`); + return match[1]!; + }; + const dir = bound(/^- ([A-Z][A-Z_]+): [^\n]*probe directory/m, 'probe directory'); + const names = { + dir, + deadline: bound(new RegExp(`([A-Z][A-Z_]+): \`${dir}/deadline\\.json\``), 'deadline file'), + guard: bound(/^- ([A-Z][A-Z_]+) = `[^`\n]*\/gstack-qa-deadline`/m, 'gstack-qa-deadline'), + recorder: bound(/^- ([A-Z][A-Z_]+) = `[^`\n]*\/gstack-qa-evidence`/m, 'gstack-qa-evidence'), + }; + if (new Set(Object.values(names)).size !== 4) throw new Error(`QA probe names must be distinct: ${JSON.stringify(names)}`); + return names; +} diff --git a/test/qa-browser-preservation.test.ts b/test/qa-browser-preservation.test.ts index 3d3f9604c..0d137b4ec 100644 --- a/test/qa-browser-preservation.test.ts +++ b/test/qa-browser-preservation.test.ts @@ -8,6 +8,7 @@ import { generateQAExploratory } from '../scripts/resolvers/qa'; import { generateTestBootstrap } from '../scripts/resolvers/testing'; import { HOST_PATHS } from '../scripts/resolvers/types'; import { runBashScript } from './helpers/bash-script'; +import { qaProbeNames } from './helpers/qa-probe-names'; const ctx = { host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude }; const method = generateQAMethodology(ctx); @@ -149,16 +150,17 @@ describe('compact QA browser recipes retain native operations', () => { test('bounded exploration rechecks the clock around checkpoints without replacing probe evidence', () => { for (const skillName of ['qa', 'qa-only', 'review', 'ship']) { const loop = generateQAExploratory({ ...ctx, skillName }); + const n = qaProbeNames(loop); for (const contract of [ - 'bun G start D SECONDS [EARLIER_UTC]', + `bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`, 'Set SECONDS to the shorter mode/caller limit', - 'G enforces the deadline', + `${n.guard} enforces the deadline`, 'QA_DEADLINE receipts are not observations', - 'Never reset D/bypass G', 'Report refusals as not-run', - 'Bounded browsers: `bun G run D -- COMMAND ARGS`', + `\`bun ${n.guard} run ${n.deadline} -- COMMAND ARGS\``, 'announce finite command timeouts', ]) expect(loop).toContain(contract); + expect(loop).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i')); for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(loop).toContain(`${field}:`); expect(loop).toContain('Functional Full, Quick and Regression have no default total timer'); if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget'); diff --git a/test/qa-caller-authority.test.ts b/test/qa-caller-authority.test.ts index 2273b63f5..72d68db1e 100644 --- a/test/qa-caller-authority.test.ts +++ b/test/qa-caller-authority.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { ALL_HOST_CONFIGS } from '../hosts'; import { RESOLVERS } from '../scripts/resolvers'; import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; +import { qaProbeNames } from './helpers/qa-probe-names'; const root = join(import.meta.dir, '..'); const callers = ['qa', 'qa-only', 'review', 'ship']; @@ -76,9 +77,11 @@ describe('QA caller authority in pure host renders', () => { for (const caller of callers) { const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller)); expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)'); - expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]'); - expect(body).toContain('G enforces the deadline'); - expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success')); + const n = qaProbeNames(body); + const start = `bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`; + expect(body).toContain(start); + expect(body).toContain(`${n.guard} enforces the deadline`); + expect(body.indexOf(start)).toBeLessThan(body.indexOf('1. First demonstrate success')); expect(body).toMatch(/stop when scoped contracts are tested or blocked/i); } }); diff --git a/test/qa-exploratory-callers.test.ts b/test/qa-exploratory-callers.test.ts index b1ed75eb3..da92ebee6 100644 --- a/test/qa-exploratory-callers.test.ts +++ b/test/qa-exploratory-callers.test.ts @@ -15,6 +15,7 @@ import { CAPTURE_MS } from './helpers/eval-budgets'; import { readQACheckpointFiles } from './helpers/qa-checkpoint-evidence'; import { generateQAExploratory, generateQAResource, generateQAReview, generateQAReviewPreflight } from '../scripts/resolvers/qa'; import { HOST_PATHS } from '../scripts/resolvers/types'; +import { qaProbeNames } from './helpers/qa-probe-names'; function nativeCall(id: string, name: string, input: object, output: string, parent: string | null = null, failed = false) { return [ @@ -673,10 +674,12 @@ describe('generated actual parent paths', () => { expect(text).not.toContain('nest unchanged child JSON'); }); test('the shared smoke has explicit limits without waiving required plan checks', () => { - const body = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8').replace(/\s+/g, ' '); + const raw = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8'); + const body = raw.replace(/\s+/g, ' '); expect(body).toMatch(/stop after 5 minutes or 12 probes/i); - expect(body).toContain('G enforces the deadline'); - expect(body).toMatch(/never reset D\/bypass G/i); + const n = qaProbeNames(raw); + expect(body).toContain(`${n.guard} enforces the deadline`); + expect(body).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i')); expect(body).toMatch(/plan checks and revalidation remain required beyond this smoke budget/i); expect(body).toContain('leaves /review incomplete'); expect(body).toMatch(/\/ship blocked unless the user explicitly accepts that named risk/i); diff --git a/test/qa-probe-gates.test.ts b/test/qa-probe-gates.test.ts index 59ece4a61..d3cf357e4 100644 --- a/test/qa-probe-gates.test.ts +++ b/test/qa-probe-gates.test.ts @@ -5,6 +5,9 @@ import { ALL_HOST_CONFIGS } from '../hosts'; import { generateQAExploratory, generateQAMethodReads, generateQAReview, generateQAReviewPreflight } from '../scripts/resolvers/qa'; import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates'; import { HOST_PATHS } from '../scripts/resolvers/types'; +import { qaProbeNames } from './helpers/qa-probe-names'; + +const startCommand = (text: string) => { const n = qaProbeNames(text); return `bun ${n.guard} start ${n.deadline} SECONDS`; }; function assertPreparation(text: string) { expect(text).toMatch(/reads in order before writing charters or probing/i); @@ -12,7 +15,7 @@ function assertPreparation(text: string) { expect(text).toMatch(/do not repeat a Read already completed/i); const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`', '**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight', - 'bun G start D SECONDS', '1. First demonstrate success']; + startCommand(text), '1. First demonstrate success']; const positions = stages.map(stage => text.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); @@ -26,13 +29,13 @@ function assertBoundsAndLayout(text: string) { for (const rule of [ /functional full, quick and regression have no default total timer/i, /shorter mode\/caller limit/i, - /without a total time limit, do not start D/i, + new RegExp(`without a total time limit, do not (?:start|create) ${qaProbeNames(text).deadline}`, 'i'), /finite command timeouts/i, /mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i, /one final report at REPORT_DIR/i, /caller paths win/i, ]) expect(text).toMatch(rule); - expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS')); + expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf(startCommand(text))); } function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) { @@ -127,7 +130,7 @@ describe('QA probe entry and checkpoint gates', () => { 'QA_DEADLINE receipts are not observations', ]) expect(text).toContain(contract); assertBoundsAndLayout(text); - expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf('bun G start D')); + expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf(startCommand(text))); } }); @@ -152,7 +155,7 @@ describe('QA probe entry and checkpoint gates', () => { } else { expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i); } - expect(text).toMatch(/Q supplies observed; never transcribe it/i); + expect(text).toMatch(new RegExp(`${qaProbeNames(text).recorder} supplies observed; never transcribe it`, 'i')); } } }); @@ -163,15 +166,19 @@ describe('QA probe entry and checkpoint gates', () => { const steps = text.slice(text.indexOf('1. First demonstrate success')).split(/\n(?=[2-5]\. )/); expect(steps).toHaveLength(5); expect(text).toContain('/gstack-qa-deadline'); - expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]'); - expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`'); - expect(text).toMatch(/never reset D\/bypass G/i); - expect(text).toMatch(/invalid\/missing D stops probes/i); + const n = qaProbeNames(text); + for (const name of Object.values(n)) expect(name.length).toBeGreaterThan(3); + expect(text).toContain(`bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`); + expect(text).toContain(`\`bun ${n.guard} run ${n.deadline} -- COMMAND ARGS\``); + expect(text).toContain(`\`bun ${n.recorder} capture ${n.dir} NNN [--public] --deadline ${n.deadline} -- COMMAND ARGS\``); + expect(text).toContain(`\`bun ${n.recorder} materialize ${n.dir} annotations.json\``); + expect(text).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i')); + expect(text).toMatch(new RegExp(`invalid/missing ${n.deadline} stops probes`, 'i')); expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i); - expect(steps[1]).toContain('`bun G status D`'); + expect(steps[1]).toContain(`\`bun ${n.guard} status ${n.deadline}\``); expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**')); expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i); - expect(steps[2]).toContain('G enforces the deadline'); + expect(steps[2]).toContain(`${n.guard} enforces the deadline`); expect(steps[2]).toMatch(/refusals as not-run/i); expect(steps[3]).toContain('via steps 2–3'); expect(steps[4]).toContain('return to step 2');