mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
C2: Browser Quick gets a feasible budget; QA loop names its directory and stop path
Judge rationales on the qa-only panels (2 of the last 3) cited the 30 s Browser Quick budget: homepage + 5 targets, each a checkpoint write and a guarded command, left ~5 s per probe. Quick is now 180 s (method text: 3 minutes); the qa-only-no-fix paid check expects the same native deadline. PROBE_DIR points at its source line, a stopped loop goes to the final report steps (annotations, materialize, Markdown), and /review and /ship say that after smoke expiry only plan checks rerun in revalidation.
This commit is contained in:
1 parent
47e86ce63b
commit
78d4a2a343
6 files changed
+51
-12
No files matched your search
@@ -73,14 +73,14 @@ ${reportOnly ? '' : `For /review and /ship, no plan/server is required.
|
||||
Stop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces).
|
||||
Explicit plan checks and revalidation remain required beyond this smoke budget.`}
|
||||
For /qa and /qa-only:
|
||||
- Browser Quick: SECONDS=30. Browser Full/Regression: SECONDS=900.
|
||||
- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900.
|
||||
- Functional Full, Quick and Regression have no default total timer.
|
||||
Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.
|
||||
Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.
|
||||
Stop when scoped contracts are tested or blocked.
|
||||
Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.
|
||||
Names used below (quote every path):
|
||||
- PROBE_DIR: this surface's owned probe directory, chosen above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run.
|
||||
- PROBE_DIR: this surface's probe directory from the line above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run.
|
||||
- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.
|
||||
- DEADLINE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates DEADLINE_FILE, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline.
|
||||
- EVIDENCE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`PROBE_DIR/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`.
|
||||
@@ -103,7 +103,7 @@ How one probe works: (1) if bounded, check the clock and choose the next command
|
||||
|
||||
1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.
|
||||
2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`.
|
||||
If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint.
|
||||
If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint: go to §4 (annotations, materialize, then Markdown).
|
||||
${reportOnly ? ` **Classify the last result before copying it.** For public or synthetic observations,
|
||||
retain the entire result unchanged, including owned fixture paths, IDs, hashes and
|
||||
existing credential placeholders. An absolute state path is not itself a secret.
|
||||
@@ -263,7 +263,7 @@ Never install, import cookies or bootstrap tests. Functional-only skips browser
|
||||
|
||||
**3. Run smoke and plan checks.**
|
||||
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. Post-expiry smoke rechecks are not-run.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. After smoke expiry, step 4c reruns plan checks only; affected smoke checks are not-run.
|
||||
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.${ship ? '' : ' /review sets none; only an invoker-supplied EARLIER_UTC counts.'}
|
||||
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
|
||||
|
||||
|
||||
@@ -169,7 +169,7 @@ Use the supplied URL or first responder/staging/preview; ask if none. Test chang
|
||||
Visit every reachable page (5-15 minutes). Score health; document 5-10 evidenced issues, never invent any.
|
||||
|
||||
### Quick (\`--quick\`)
|
||||
30 seconds: homepage + top 5 navigation targets. Check loads/console/broken links; score per Health Score Rubric; skip detailed issues/checklist, never the shared loop's gates.
|
||||
3 minutes: homepage + top 5 navigation targets. Check loads/console/broken links; score per Health Score Rubric; skip detailed issues/checklist, never the shared loop's gates.
|
||||
|
||||
### Regression (\`--regression <baseline>\`)
|
||||
Run Full; append fixed/new issues and score delta. Preserve the supplied prior baseline.
|
||||
|
||||
@@ -157,7 +157,7 @@ export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick<
|
||||
|
||||
export function assertQaBrowserDeadline(calls: Call[], options: Options & { expectedBudgetMs?: number }) {
|
||||
const fail = (reason: string): never => { throw new Error('QA deadline: ' + reason); };
|
||||
const { directory, guard, browse, started, ended, expectedBudgetMs = 30000 } = options;
|
||||
const { directory, guard, browse, started, ended, expectedBudgetMs = 180_000 } = options;
|
||||
if (!Number.isSafeInteger(expectedBudgetMs) || expectedBudgetMs <= 0 || expectedBudgetMs > 2_147_483_647) fail('invalid expected deadline budget');
|
||||
const { file, allowed } = qaDeadlineShellPolicy(directory, guard, browse);
|
||||
const reportRoot = path.join(directory, 'qa-reports');
|
||||
|
||||
@@ -14,7 +14,7 @@ const directories: string[] = [];
|
||||
const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
afterEach(() => { for (const directory of directories.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); });
|
||||
|
||||
function fixture(expectedBudgetMs = 30000) {
|
||||
function fixture(expectedBudgetMs = 180_000) {
|
||||
const started = Date.now();
|
||||
const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-gate-'));
|
||||
directories.push(directory);
|
||||
@@ -218,7 +218,7 @@ test('child-emitted receipt forgery and direct state reset scripts are rejected'
|
||||
expect(() => f.check()).toThrow('reserved deadline evidence');
|
||||
});
|
||||
|
||||
test('short fixture deadlines require an explicit contract; the native default remains exactly 30s', () => {
|
||||
test('short fixture deadlines require an explicit contract; the native default is the 180s Browser Quick budget', () => {
|
||||
const native = fixture();
|
||||
native.run();
|
||||
expect(assertQaBrowserDeadline(native.calls, { directory: native.directory, guard, browse,
|
||||
@@ -227,7 +227,7 @@ test('short fixture deadlines require an explicit contract; the native default r
|
||||
f.run();
|
||||
expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 });
|
||||
expect(() => assertQaBrowserDeadline(f.calls, { directory: f.directory, guard, browse,
|
||||
started: f.started, ended: Date.now() })).toThrow('expected 30000ms deadline');
|
||||
started: f.started, ended: Date.now() })).toThrow('expected 180000ms deadline');
|
||||
for (const expectedBudgetMs of [0, -1, 0.5, 2_147_483_648, NaN, Infinity]) {
|
||||
expect(() => assertQaBrowserDeadline(f.calls, { directory: f.directory, guard, browse,
|
||||
started: f.started, ended: Date.now(), expectedBudgetMs })).toThrow('invalid expected deadline budget');
|
||||
|
||||
@@ -180,7 +180,7 @@ describe('compact QA browser recipes retain native operations', () => {
|
||||
'Resolve conflicting depth flags by asking before probes',
|
||||
'Diff-aware selects scope, not another pass',
|
||||
'After selecting and isolating a browser surface',
|
||||
'Visit every reachable page (5-15 minutes)', '30 seconds: homepage + top 5 navigation targets',
|
||||
'Visit every reachable page (5-15 minutes)', '3 minutes: homepage + top 5 navigation targets',
|
||||
"skip detailed issues/checklist, never the shared loop's gates",
|
||||
]) expect(prose).toContain(contract);
|
||||
const titles = ['Initialize', 'Authenticate (if needed)', 'Orient', 'Explore', 'Document', 'Wrap Up'];
|
||||
|
||||
@@ -25,7 +25,7 @@ function assertPreparation(text: string) {
|
||||
}
|
||||
|
||||
function assertBoundsAndLayout(text: string) {
|
||||
for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract);
|
||||
for (const contract of ['Browser Quick: SECONDS=180', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract);
|
||||
for (const rule of [
|
||||
/functional full, quick and regression have no default total timer/i,
|
||||
/shorter mode\/caller limit/i,
|
||||
@@ -130,7 +130,7 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
'QA_DEADLINE receipts are not observations',
|
||||
]) expect(text).toContain(contract);
|
||||
assertBoundsAndLayout(text);
|
||||
expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf(startCommand(text)));
|
||||
expect(text.indexOf('Browser Quick: SECONDS=180')).toBeLessThan(text.indexOf(startCommand(text)));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -295,3 +295,42 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
expect(text).toMatch(/method Reads and the shared probe loop/i);
|
||||
});
|
||||
});
|
||||
|
||||
// Lane S C2: gaps the qa-only/review/ship workflow judges named in their rationales.
|
||||
describe('judge-named QA workflow gaps (C2)', () => {
|
||||
const ctx = (skillName: string) => ({ host: 'claude' as const, skillName, tmplPath: '', paths: HOST_PATHS.claude });
|
||||
|
||||
test('Browser Quick leaves time for its six page probes, and the method states the same budget', () => {
|
||||
for (const skillName of ['qa', 'qa-only']) {
|
||||
const seconds = Number(/Browser Quick: SECONDS=(\d+)\./.exec(generateQAExploratory(ctx(skillName)))?.[1]);
|
||||
// homepage + top 5 targets, each a checkpoint write plus one guarded command: 30 s left ~5 s per probe.
|
||||
expect(seconds / 6).toBeGreaterThanOrEqual(20);
|
||||
const patterns = fs.readFileSync(path.join(import.meta.dir, '..', 'qa', 'sections', 'qa-patterns.md'), 'utf8');
|
||||
expect(patterns).toContain(`${seconds / 60} minutes: homepage + top 5 navigation targets`);
|
||||
}
|
||||
});
|
||||
|
||||
test('PROBE_DIR names its source line, and stopping goes to the final report steps', () => {
|
||||
for (const skillName of ['qa', 'qa-only']) {
|
||||
const text = generateQAExploratory(ctx(skillName));
|
||||
const layout = text.indexOf('mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional');
|
||||
const probeDir = text.indexOf("PROBE_DIR: this surface's probe directory from the line above.");
|
||||
expect(layout).toBeGreaterThan(-1);
|
||||
expect(probeDir).toBeGreaterThan(layout);
|
||||
expect(text.slice(layout, probeDir).split('\n')).toHaveLength(3);
|
||||
expect(text).toContain('write the report, not a checkpoint: go to §4 (annotations, materialize, then Markdown)');
|
||||
const final = text.indexOf('## 4. Final report');
|
||||
expect(final).toBeGreaterThan(-1);
|
||||
expect(text.indexOf('annotations.json', final)).toBeLessThan(text.indexOf('materialize PROBE_DIR', final));
|
||||
}
|
||||
});
|
||||
|
||||
test('/review and /ship: after smoke expiry only plan checks rerun in revalidation', () => {
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
const text = generateQAReview(ctx(skillName));
|
||||
expect(text).toContain('After smoke expiry, step 4c reruns plan checks only; affected smoke checks are not-run.');
|
||||
expect(text).toContain('c. Re-review changed or uncertain coverage and repeat step 3 for affected checks.');
|
||||
expect(text).not.toContain('Post-expiry smoke rechecks are not-run.');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user