diff --git a/test/helpers/qa-browser-deadline-evidence.ts b/test/helpers/qa-browser-deadline-evidence.ts index 2da105051..512dc6c43 100644 --- a/test/helpers/qa-browser-deadline-evidence.ts +++ b/test/helpers/qa-browser-deadline-evidence.ts @@ -105,18 +105,33 @@ export function assertQaBrowserPreparation(transcript: unknown[], options: Pick< if (!prepared) throw new Error('QA preparation: owned nonempty report Write must complete before guard start/baseline'); } -export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick) { +/** + * The fixture prompt routes browser cleanup through the guard, and a probe + * excludes bookkeeping. Only closing the tab with the supplied browse binary, + * as the whole child, is cleanup; anything else in the child is a probe. + */ +function browserCleanupChild(child: string[], browse: string): boolean { + if (child.length === 2) return child[0] === browse && child[1] === 'closetab'; + return child.length === 3 && child[0] === 'bash' && child[1] === '-c' + && [`B="${browse}"; "$B" closetab`, `"${browse}" closetab`].includes(child[2].trim()); +} + +export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick) { const fail = (reason: string): never => { throw new Error('QA checkpoint: ' + reason); }; const root = path.join(options.directory, 'qa-reports'); const files = readQACheckpointFiles(root); const calls = browserNativeCalls(transcript); - const runs = calls.filter(call => { - if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return false; + const guarded = calls.flatMap(call => { + if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return []; let argv: string[]; - try { argv = literalArgv(call.input.command); } catch { return false; } + try { argv = literalArgv(call.input.command); } catch { return []; } return ['bun', process.execPath].includes(argv[0]) && argv[1] === options.guard - && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json'); + && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json') + ? [{ call, cleanup: argv[4] === '--' && browserCleanupChild(argv.slice(5), options.browse) }] : []; }); + const runs = guarded.filter(run => !run.cleanup).map(run => run.call); + const lastProbe = runs.at(-1); + if (lastProbe && guarded.some(run => run.cleanup && run.call.start < lastProbe.end)) fail('browser cleanup preceded a probe'); const notes: Array<{ call: (typeof calls)[number]; value: any }> = []; const written = new Set(); for (const call of calls) { diff --git a/test/qa-browser-deadline-evidence.test.ts b/test/qa-browser-deadline-evidence.test.ts index ced94fdcf..5a019d8ad 100644 --- a/test/qa-browser-deadline-evidence.test.ts +++ b/test/qa-browser-deadline-evidence.test.ts @@ -58,11 +58,37 @@ test.each(['captured-rewrite', 'verbatim-text', 'decoded-json', 'wrapped-json', fs.writeFileSync(write.input.file_path, scenario === 'disk-mismatch' ? '{}' : write.input.content); if (scenario === 'missing-ack') events.splice(3, 1); if (scenario === 'late-write') events.push(...events.splice(2, 2)); - const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard }); + const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse }); if (scenario === 'verbatim-text' || scenario === 'decoded-json') expect(check).not.toThrow(); else expect(check).toThrow(); }); +// CI run 37252022510 (qa-only-no-fix): after its last probe the agent closed the tab +// through the guard, as the fixture prompt directs, and the clock had expired. +test.each(['cleanup-after-last-probe', 'cleanup-direct-argv', 'probe-after-cleanup', 'compound-cleanup', 'other-browser-cleanup']) + ('guarded browser cleanup after the last probe needs no checkpoint; anything more is a probe: %s', scenario => { + const f = fixture(); + const events = JSON.parse(JSON.stringify(capturedObservation.events) + .replaceAll('__QA_FIXTURE__', f.directory).replaceAll('__QA_GUARD__', guard).replaceAll('__QA_BROWSE__', browse)); + const baseline = events[1].message.content[0], write = events[2].message.content[0]; + const lines = baseline.content.split('\n'); + const first = lines.findIndex((line: string) => line.startsWith('QA_DEADLINE ')); + const last = lines.findLastIndex((line: string) => line.startsWith('QA_DEADLINE ')); + write.input.content = JSON.stringify({ ...JSON.parse(write.input.content), observed: lines.slice(first + 1, last).join('\n') }); + fs.writeFileSync(write.input.file_path, write.input.content); + const child = scenario === 'cleanup-direct-argv' ? quote(browse) + ' closetab' + : `bash -c 'B="${scenario === 'other-browser-cleanup' ? '/usr/bin/true' : browse}"; "$B" closetab${scenario === 'compound-cleanup' ? '; "$B" goto http://127.0.0.1:1/' : ''}'`; + const expired = events[5].message.content[0].content; + const cleanup = [ + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'toolu_cleanup', name: 'Bash', input: { command: `bun ${quote(guard)} run ${quote(path.join(f.directory, 'qa-reports/deadline.json'))} -- ${child}` } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'toolu_cleanup', content: expired, is_error: true }] } }, + ]; + if (scenario === 'probe-after-cleanup') events.splice(4, 0, ...cleanup); else events.push(...cleanup); + const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse }); + if (['cleanup-after-last-probe', 'cleanup-direct-argv'].includes(scenario)) expect(check).not.toThrow(); + else expect(check).toThrow(scenario === 'probe-after-cleanup' ? 'browser cleanup preceded a probe' : 'follow-up lacks one acknowledged preceding checkpoint'); + }); + test('R70 retained public ordering has no completed report Write before its clock/baseline', () => { const f = fixture(); const events = JSON.parse(JSON.stringify(capturedPreparation.events) diff --git a/test/skill-e2e-qa-workflow.test.ts b/test/skill-e2e-qa-workflow.test.ts index 57d8e8988..d2ebc644b 100644 --- a/test/skill-e2e-qa-workflow.test.ts +++ b/test/skill-e2e-qa-workflow.test.ts @@ -188,7 +188,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`, directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), }); assertQaBrowserCheckpoints(result.transcript, { - directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), + directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), browse: browseBin, }); passed = true; } catch (error) { diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 9cfa011eb..693f1a719 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -669,6 +669,12 @@ describeIfSelected('Other skill evals', [ judgeContext: 'a pre-landing review with bounded exploratory QA', judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs', agentCapability: 'frontier', + // This ~64k-token bundle is the largest workflow input. At default effort, thinking reached 6.9k output + // tokens and 88 s in 24 measured samples, and a CI sample spent all 8192 tokens without an answer. + // Medium kept 24 of 24 under 5k tokens and 63 s with the same gating scores; the budget is no cap. + maxTokens: 65_536, + stream: true, + effort: 'medium', }); }, WORKFLOW_JUDGE_TEST_MS); diff --git a/test/workflow-judge-cache.test.ts b/test/workflow-judge-cache.test.ts index a049470a1..f13534be5 100644 --- a/test/workflow-judge-cache.test.ts +++ b/test/workflow-judge-cache.test.ts @@ -399,7 +399,13 @@ test('Ship sends its authorized 64k cap and compact response contract through th expect(options.maxTokens).toBe(65_536); expect(options.stream).toBe(true); expect(options.effort).toBe('medium'); - expect(source.match(/^\s+effort: '/gm)).toHaveLength(1); + // Only measured registrations bound judge effort: ship and the ~64k-token review bundle. + expect(source.match(/^\s+effort: '/gm)).toHaveLength(2); + const review = source.match(/testIfSelected\('review\/SKILL\.md workflow',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/); + expect(review).not.toBeNull(); + const reviewOptions = new Function('QA_DISCOVERY_REFERENCES', `return ({${review![1]}});`)(QA_DISCOVERY_REFERENCES); + expect([reviewOptions.maxTokens, reviewOptions.stream, reviewOptions.effort, reviewOptions.schemaTransport, reviewOptions.compactReasoning]) + .toEqual([65_536, true, 'medium', undefined, undefined]); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern'); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength'); expect(source.match(/compactReasoning: true/g)).toHaveLength(1);