test(evals): bound the review judge's thinking; guarded browser cleanup is not a probe

review/SKILL.md workflow: the ~64k-token bundle let default-effort thinking
spend all 8192 output tokens without an answer in CI. Medium effort with a
64k streaming cap kept 24/24 measured samples under 5k tokens and 63 s with
the same gating scores (as ship already does).

qa-only-no-fix: the fixture prompt routes browser cleanup through the deadline
guard, but the checkpoint detector treated every guarded run as a probe that
needs a checkpoint. A child that only closes the tab with the supplied browse
binary, after the last probe, is cleanup; anything else stays a probe. The
captured CI transcript (run 37252022510) now passes and new scenarios pin
the negative controls.

test/helpers/qa-browser-deadline-evidence.ts

@@ -105,18 +105,33 @@ export function assertQaBrowserPreparation(transcript: unknown[], options: Pick<
   if (!prepared) throw new Error('QA preparation: owned nonempty report Write must complete before guard start/baseline');
 }
 
-export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick<Options, 'directory' | 'guard'>) {
+/**
+ * The fixture prompt routes browser cleanup through the guard, and a probe
+ * excludes bookkeeping. Only closing the tab with the supplied browse binary,
+ * as the whole child, is cleanup; anything else in the child is a probe.
+ */
+function browserCleanupChild(child: string[], browse: string): boolean {
+  if (child.length === 2) return child[0] === browse && child[1] === 'closetab';
+  return child.length === 3 && child[0] === 'bash' && child[1] === '-c'
+    && [`B="${browse}"; "$B" closetab`, `"${browse}" closetab`].includes(child[2].trim());
+}
+
+export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick<Options, 'directory' | 'guard' | 'browse'>) {
   const fail = (reason: string): never => { throw new Error('QA checkpoint: ' + reason); };
   const root = path.join(options.directory, 'qa-reports');
   const files = readQACheckpointFiles(root);
   const calls = browserNativeCalls(transcript);
-  const runs = calls.filter(call => {
-    if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return false;
+  const guarded = calls.flatMap(call => {
+    if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return [];
     let argv: string[];
-    try { argv = literalArgv(call.input.command); } catch { return false; }
+    try { argv = literalArgv(call.input.command); } catch { return []; }
     return ['bun', process.execPath].includes(argv[0]) && argv[1] === options.guard
-      && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json');
+      && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json')
+      ? [{ call, cleanup: argv[4] === '--' && browserCleanupChild(argv.slice(5), options.browse) }] : [];
   });
+  const runs = guarded.filter(run => !run.cleanup).map(run => run.call);
+  const lastProbe = runs.at(-1);
+  if (lastProbe && guarded.some(run => run.cleanup && run.call.start < lastProbe.end)) fail('browser cleanup preceded a probe');
   const notes: Array<{ call: (typeof calls)[number]; value: any }> = [];
   const written = new Set<string>();
   for (const call of calls) {

test/qa-browser-deadline-evidence.test.ts

@@ -58,11 +58,37 @@ test.each(['captured-rewrite', 'verbatim-text', 'decoded-json', 'wrapped-json',
     fs.writeFileSync(write.input.file_path, scenario === 'disk-mismatch' ? '{}' : write.input.content);
     if (scenario === 'missing-ack') events.splice(3, 1);
     if (scenario === 'late-write') events.push(...events.splice(2, 2));
-    const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard });
+    const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse });
     if (scenario === 'verbatim-text' || scenario === 'decoded-json') expect(check).not.toThrow();
     else expect(check).toThrow();
   });
 
+// CI run 37252022510 (qa-only-no-fix): after its last probe the agent closed the tab
+// through the guard, as the fixture prompt directs, and the clock had expired.
+test.each(['cleanup-after-last-probe', 'cleanup-direct-argv', 'probe-after-cleanup', 'compound-cleanup', 'other-browser-cleanup'])
+  ('guarded browser cleanup after the last probe needs no checkpoint; anything more is a probe: %s', scenario => {
+    const f = fixture();
+    const events = JSON.parse(JSON.stringify(capturedObservation.events)
+      .replaceAll('__QA_FIXTURE__', f.directory).replaceAll('__QA_GUARD__', guard).replaceAll('__QA_BROWSE__', browse));
+    const baseline = events[1].message.content[0], write = events[2].message.content[0];
+    const lines = baseline.content.split('\n');
+    const first = lines.findIndex((line: string) => line.startsWith('QA_DEADLINE '));
+    const last = lines.findLastIndex((line: string) => line.startsWith('QA_DEADLINE '));
+    write.input.content = JSON.stringify({ ...JSON.parse(write.input.content), observed: lines.slice(first + 1, last).join('\n') });
+    fs.writeFileSync(write.input.file_path, write.input.content);
+    const child = scenario === 'cleanup-direct-argv' ? quote(browse) + ' closetab'
+      : `bash -c 'B="${scenario === 'other-browser-cleanup' ? '/usr/bin/true' : browse}"; "$B" closetab${scenario === 'compound-cleanup' ? '; "$B" goto http://127.0.0.1:1/' : ''}'`;
+    const expired = events[5].message.content[0].content;
+    const cleanup = [
+      { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'toolu_cleanup', name: 'Bash', input: { command: `bun ${quote(guard)} run ${quote(path.join(f.directory, 'qa-reports/deadline.json'))} -- ${child}` } }] } },
+      { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'toolu_cleanup', content: expired, is_error: true }] } },
+    ];
+    if (scenario === 'probe-after-cleanup') events.splice(4, 0, ...cleanup); else events.push(...cleanup);
+    const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse });
+    if (['cleanup-after-last-probe', 'cleanup-direct-argv'].includes(scenario)) expect(check).not.toThrow();
+    else expect(check).toThrow(scenario === 'probe-after-cleanup' ? 'browser cleanup preceded a probe' : 'follow-up lacks one acknowledged preceding checkpoint');
+  });
+
 test('R70 retained public ordering has no completed report Write before its clock/baseline', () => {
   const f = fixture();
   const events = JSON.parse(JSON.stringify(capturedPreparation.events)

test/skill-e2e-qa-workflow.test.ts

@@ -188,7 +188,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
         directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'),
       });
       assertQaBrowserCheckpoints(result.transcript, {
-        directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'),
+        directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), browse: browseBin,
       });
       passed = true;
     } catch (error) {

test/skill-llm-eval.test.ts

@@ -669,6 +669,12 @@ describeIfSelected('Other skill evals', [
       judgeContext: 'a pre-landing review with bounded exploratory QA',
       judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs',
       agentCapability: 'frontier',
+      // This ~64k-token bundle is the largest workflow input. At default effort, thinking reached 6.9k output
+      // tokens and 88 s in 24 measured samples, and a CI sample spent all 8192 tokens without an answer.
+      // Medium kept 24 of 24 under 5k tokens and 63 s with the same gating scores; the budget is no cap.
+      maxTokens: 65_536,
+      stream: true,
+      effort: 'medium',
     });
   }, WORKFLOW_JUDGE_TEST_MS);
 

test/workflow-judge-cache.test.ts

@@ -399,7 +399,13 @@ test('Ship sends its authorized 64k cap and compact response contract through th
   expect(options.maxTokens).toBe(65_536);
   expect(options.stream).toBe(true);
   expect(options.effort).toBe('medium');
-  expect(source.match(/^\s+effort: '/gm)).toHaveLength(1);
+  // Only measured registrations bound judge effort: ship and the ~64k-token review bundle.
+  expect(source.match(/^\s+effort: '/gm)).toHaveLength(2);
+  const review = source.match(/testIfSelected\('review\/SKILL\.md workflow',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n    \}\);/);
+  expect(review).not.toBeNull();
+  const reviewOptions = new Function('QA_DISCOVERY_REFERENCES', `return ({${review![1]}});`)(QA_DISCOVERY_REFERENCES);
+  expect([reviewOptions.maxTokens, reviewOptions.stream, reviewOptions.effort, reviewOptions.schemaTransport, reviewOptions.compactReasoning])
+    .toEqual([65_536, true, 'medium', undefined, undefined]);
   expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern');
   expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength');
   expect(source.match(/compactReasoning: true/g)).toHaveLength(1);