diff --git a/scripts/resolvers/qa.ts b/scripts/resolvers/qa.ts index 3821d4273..9b3cd9e7b 100644 --- a/scripts/resolvers/qa.ts +++ b/scripts/resolvers/qa.ts @@ -73,14 +73,14 @@ ${reportOnly ? '' : `For /review and /ship, no plan/server is required. Stop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces). Explicit plan checks and revalidation remain required beyond this smoke budget.`} For /qa and /qa-only: -- Browser Quick: SECONDS=30. Browser Full/Regression: SECONDS=900. +- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900. - Functional Full, Quick and Regression have no default total timer. Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound. Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts. Stop when scoped contracts are tested or blocked. Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win. Names used below (quote every path): -- PROBE_DIR: this surface's owned probe directory, chosen above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run. +- PROBE_DIR: this surface's probe directory from the line above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run. - SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one. - DEADLINE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates DEADLINE_FILE, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline. - EVIDENCE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`PROBE_DIR/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`. @@ -103,7 +103,7 @@ How one probe works: (1) if bounded, check the clock and choose the next command 1. First demonstrate success: output AND durable effects. Guard if bounded; await completion. 2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`. - If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint. + If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint: go to §4 (annotations, materialize, then Markdown). ${reportOnly ? ` **Classify the last result before copying it.** For public or synthetic observations, retain the entire result unchanged, including owned fixture paths, IDs, hashes and existing credential placeholders. An absolute state path is not itself a secret. @@ -263,7 +263,7 @@ Never install, import cookies or bootstrap tests. Functional-only skips browser **3. Run smoke and plan checks.** Follow the shared Probe loop for smoke checks and replays until the smoke limit. -Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. Post-expiry smoke rechecks are not-run. +Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. After smoke expiry, step 4c reruns plan checks only; affected smoke checks are not-run. Use finite command timeouts, capped at the caller's remaining time if it has a deadline.${ship ? '' : ' /review sets none; only an invoker-supplied EARLIER_UTC counts.'} Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run. diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index ca8db8f0d..d5ffaf65b 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -169,7 +169,7 @@ Use the supplied URL or first responder/staging/preview; ask if none. Test chang Visit every reachable page (5-15 minutes). Score health; document 5-10 evidenced issues, never invent any. ### Quick (\`--quick\`) -30 seconds: homepage + top 5 navigation targets. Check loads/console/broken links; score per Health Score Rubric; skip detailed issues/checklist, never the shared loop's gates. +3 minutes: homepage + top 5 navigation targets. Check loads/console/broken links; score per Health Score Rubric; skip detailed issues/checklist, never the shared loop's gates. ### Regression (\`--regression \`) Run Full; append fixed/new issues and score delta. Preserve the supplied prior baseline. diff --git a/test/helpers/qa-browser-deadline-evidence.ts b/test/helpers/qa-browser-deadline-evidence.ts index 2da105051..fd71151bb 100644 --- a/test/helpers/qa-browser-deadline-evidence.ts +++ b/test/helpers/qa-browser-deadline-evidence.ts @@ -157,7 +157,7 @@ export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick< export function assertQaBrowserDeadline(calls: Call[], options: Options & { expectedBudgetMs?: number }) { const fail = (reason: string): never => { throw new Error('QA deadline: ' + reason); }; - const { directory, guard, browse, started, ended, expectedBudgetMs = 30000 } = options; + const { directory, guard, browse, started, ended, expectedBudgetMs = 180_000 } = options; if (!Number.isSafeInteger(expectedBudgetMs) || expectedBudgetMs <= 0 || expectedBudgetMs > 2_147_483_647) fail('invalid expected deadline budget'); const { file, allowed } = qaDeadlineShellPolicy(directory, guard, browse); const reportRoot = path.join(directory, 'qa-reports'); diff --git a/test/qa-browser-deadline-evidence.test.ts b/test/qa-browser-deadline-evidence.test.ts index ced94fdcf..8ff36ce15 100644 --- a/test/qa-browser-deadline-evidence.test.ts +++ b/test/qa-browser-deadline-evidence.test.ts @@ -14,7 +14,7 @@ const directories: string[] = []; const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; afterEach(() => { for (const directory of directories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); }); -function fixture(expectedBudgetMs = 30000) { +function fixture(expectedBudgetMs = 180_000) { const started = Date.now(); const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-gate-')); directories.push(directory); @@ -218,7 +218,7 @@ test('child-emitted receipt forgery and direct state reset scripts are rejected' expect(() => f.check()).toThrow('reserved deadline evidence'); }); -test('short fixture deadlines require an explicit contract; the native default remains exactly 30s', () => { +test('short fixture deadlines require an explicit contract; the native default is the 180s Browser Quick budget', () => { const native = fixture(); native.run(); expect(assertQaBrowserDeadline(native.calls, { directory: native.directory, guard, browse, @@ -227,7 +227,7 @@ test('short fixture deadlines require an explicit contract; the native default r f.run(); expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 }); expect(() => assertQaBrowserDeadline(f.calls, { directory: f.directory, guard, browse, - started: f.started, ended: Date.now() })).toThrow('expected 30000ms deadline'); + started: f.started, ended: Date.now() })).toThrow('expected 180000ms deadline'); for (const expectedBudgetMs of [0, -1, 0.5, 2_147_483_648, NaN, Infinity]) { expect(() => assertQaBrowserDeadline(f.calls, { directory: f.directory, guard, browse, started: f.started, ended: Date.now(), expectedBudgetMs })).toThrow('invalid expected deadline budget'); diff --git a/test/qa-browser-preservation.test.ts b/test/qa-browser-preservation.test.ts index 0d137b4ec..ecf622f8d 100644 --- a/test/qa-browser-preservation.test.ts +++ b/test/qa-browser-preservation.test.ts @@ -180,7 +180,7 @@ describe('compact QA browser recipes retain native operations', () => { 'Resolve conflicting depth flags by asking before probes', 'Diff-aware selects scope, not another pass', 'After selecting and isolating a browser surface', - 'Visit every reachable page (5-15 minutes)', '30 seconds: homepage + top 5 navigation targets', + 'Visit every reachable page (5-15 minutes)', '3 minutes: homepage + top 5 navigation targets', "skip detailed issues/checklist, never the shared loop's gates", ]) expect(prose).toContain(contract); const titles = ['Initialize', 'Authenticate (if needed)', 'Orient', 'Explore', 'Document', 'Wrap Up']; diff --git a/test/qa-probe-gates.test.ts b/test/qa-probe-gates.test.ts index d3cf357e4..c275068c5 100644 --- a/test/qa-probe-gates.test.ts +++ b/test/qa-probe-gates.test.ts @@ -25,7 +25,7 @@ function assertPreparation(text: string) { } function assertBoundsAndLayout(text: string) { - for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract); + for (const contract of ['Browser Quick: SECONDS=180', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract); for (const rule of [ /functional full, quick and regression have no default total timer/i, /shorter mode\/caller limit/i, @@ -130,7 +130,7 @@ describe('QA probe entry and checkpoint gates', () => { 'QA_DEADLINE receipts are not observations', ]) expect(text).toContain(contract); assertBoundsAndLayout(text); - expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf(startCommand(text))); + expect(text.indexOf('Browser Quick: SECONDS=180')).toBeLessThan(text.indexOf(startCommand(text))); } }); @@ -295,3 +295,42 @@ describe('QA probe entry and checkpoint gates', () => { expect(text).toMatch(/method Reads and the shared probe loop/i); }); }); + +// Lane S C2: gaps the qa-only/review/ship workflow judges named in their rationales. +describe('judge-named QA workflow gaps (C2)', () => { + const ctx = (skillName: string) => ({ host: 'claude' as const, skillName, tmplPath: '', paths: HOST_PATHS.claude }); + + test('Browser Quick leaves time for its six page probes, and the method states the same budget', () => { + for (const skillName of ['qa', 'qa-only']) { + const seconds = Number(/Browser Quick: SECONDS=(\d+)\./.exec(generateQAExploratory(ctx(skillName)))?.[1]); + // homepage + top 5 targets, each a checkpoint write plus one guarded command: 30 s left ~5 s per probe. + expect(seconds / 6).toBeGreaterThanOrEqual(20); + const patterns = fs.readFileSync(path.join(import.meta.dir, '..', 'qa', 'sections', 'qa-patterns.md'), 'utf8'); + expect(patterns).toContain(`${seconds / 60} minutes: homepage + top 5 navigation targets`); + } + }); + + test('PROBE_DIR names its source line, and stopping goes to the final report steps', () => { + for (const skillName of ['qa', 'qa-only']) { + const text = generateQAExploratory(ctx(skillName)); + const layout = text.indexOf('mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional'); + const probeDir = text.indexOf("PROBE_DIR: this surface's probe directory from the line above."); + expect(layout).toBeGreaterThan(-1); + expect(probeDir).toBeGreaterThan(layout); + expect(text.slice(layout, probeDir).split('\n')).toHaveLength(3); + expect(text).toContain('write the report, not a checkpoint: go to §4 (annotations, materialize, then Markdown)'); + const final = text.indexOf('## 4. Final report'); + expect(final).toBeGreaterThan(-1); + expect(text.indexOf('annotations.json', final)).toBeLessThan(text.indexOf('materialize PROBE_DIR', final)); + } + }); + + test('/review and /ship: after smoke expiry only plan checks rerun in revalidation', () => { + for (const skillName of ['review', 'ship']) { + const text = generateQAReview(ctx(skillName)); + expect(text).toContain('After smoke expiry, step 4c reruns plan checks only; affected smoke checks are not-run.'); + expect(text).toContain('c. Re-review changed or uncertain coverage and repeat step 3 for affected checks.'); + expect(text).not.toContain('Post-expiry smoke rechecks are not-run.'); + } + }); +});