From 78d65fbcb955cf21e38b03b4cee55833071035a1 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 5 Oct 2026 02:39:30 +0000 Subject: [PATCH] test(evals): keep the review judge at default effort; only lift its output cap Truncation was the defect: thinking and the answer share max_tokens, and the ~64k-token review bundle once spent all 8192. The registration keeps default effort and response format and gets ship's 64k streaming cap; the guard test pins exactly that. --- test/skill-llm-eval.test.ts | 7 +++---- test/workflow-judge-cache.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 693f1a719..668c59a12 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -669,12 +669,11 @@ describeIfSelected('Other skill evals', [ judgeContext: 'a pre-landing review with bounded exploratory QA', judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs', agentCapability: 'frontier', - // This ~64k-token bundle is the largest workflow input. At default effort, thinking reached 6.9k output - // tokens and 88 s in 24 measured samples, and a CI sample spent all 8192 tokens without an answer. - // Medium kept 24 of 24 under 5k tokens and 63 s with the same gating scores; the budget is no cap. + // This ~64k-token bundle is the largest workflow input. Thinking and the answer share max_tokens: + // at default effort, 24 measured samples used up to 6.9k output tokens and a CI sample spent all + // 8192 without an answer. The ship-sized streaming cap removes truncation; effort stays default. maxTokens: 65_536, stream: true, - effort: 'medium', }); }, WORKFLOW_JUDGE_TEST_MS); diff --git a/test/workflow-judge-cache.test.ts b/test/workflow-judge-cache.test.ts index f13534be5..a11f1cc70 100644 --- a/test/workflow-judge-cache.test.ts +++ b/test/workflow-judge-cache.test.ts @@ -399,13 +399,13 @@ test('Ship sends its authorized 64k cap and compact response contract through th expect(options.maxTokens).toBe(65_536); expect(options.stream).toBe(true); expect(options.effort).toBe('medium'); - // Only measured registrations bound judge effort: ship and the ~64k-token review bundle. - expect(source.match(/^\s+effort: '/gm)).toHaveLength(2); + expect(source.match(/^\s+effort: '/gm)).toHaveLength(1); + // The ~64k-token review bundle gets only ship's output cap and streaming; its effort and response format stay default. const review = source.match(/testIfSelected\('review\/SKILL\.md workflow',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/); expect(review).not.toBeNull(); const reviewOptions = new Function('QA_DISCOVERY_REFERENCES', `return ({${review![1]}});`)(QA_DISCOVERY_REFERENCES); expect([reviewOptions.maxTokens, reviewOptions.stream, reviewOptions.effort, reviewOptions.schemaTransport, reviewOptions.compactReasoning]) - .toEqual([65_536, true, 'medium', undefined, undefined]); + .toEqual([65_536, true, undefined, undefined, undefined]); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern'); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength'); expect(source.match(/compactReasoning: true/g)).toHaveLength(1);