diff --git a/test/helpers/llm-judge.ts b/test/helpers/llm-judge.ts index 0b5ef4186..ccba69a96 100644 --- a/test/helpers/llm-judge.ts +++ b/test/helpers/llm-judge.ts @@ -104,6 +104,8 @@ export interface CallJudgeOptions { signal?: AbortSignal; /** Opt-in serialization contract; callers still validate the judgment locally. */ jsonSchema?: JSONOutputFormat['schema']; + /** Adaptive-thinking effort; the judge models accept no thinking token budget. */ + effort?: 'low' | 'medium' | 'high'; } export async function callJudge( @@ -129,7 +131,9 @@ export async function callJudge( model: resolvedModel, max_tokens: maxTokens, ...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts?.jsonSchema === undefined ? {} : { output_config: { format: { type: 'json_schema' as const, schema: opts.jsonSchema } } }), + ...(opts?.jsonSchema === undefined && opts?.effort === undefined ? {} : { output_config: { + ...(opts?.jsonSchema === undefined ? {} : { format: { type: 'json_schema' as const, schema: opts.jsonSchema } }), + ...(opts?.effort === undefined ? {} : { effort: opts.effort }) } }), messages: [{ role: 'user' as const, content: prompt }], }; const makeRequest = () => opts?.stream diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index ee9607546..272fdcd2d 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -18,6 +18,7 @@ export interface WorkflowCacheOptions { structuredResponse?: boolean; maxTokens?: number; stream?: boolean; + effort?: 'medium'; env?: NodeJS.ProcessEnv; } export interface WorkflowJudgeReuse { @@ -88,6 +89,7 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): { request: opts.stream ? 'messages.stream/user' : 'messages.create/user', retries: 0, panel: { samples: JUDGE_PANEL_SAMPLES, numeric: 'mean', boolean: 'majority' }, ...(opts.stream ? { stream: true } : {}), + ...(opts.effort ? { effort: opts.effort } : {}), ...(opts.structuredResponse ? { output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } }, response_validation: { reasoning_words_below: WORKFLOW_JUDGE_REASONING_WORD_LIMIT } } : {}) }, runtime: { image: env.EVALS_CACHE_RUNTIME_ID!, bun: Bun.version, node: process.versions.node, diff --git a/test/llm-judge-stream.test.ts b/test/llm-judge-stream.test.ts index 8e5d0e43a..1087dea0f 100644 --- a/test/llm-judge-stream.test.ts +++ b/test/llm-judge-stream.test.ts @@ -45,6 +45,20 @@ test('the pinned SDK refuses a default nonstreaming 64k request before network a expect(transport).not.toHaveBeenCalled(); }); +test('effort reaches the wire beside the response schema and is absent unless requested', async () => { + transport.mockImplementation(async () => response()); + await callJudge('score the whole bundle', 'claude-fable-5-1', { + max_tokens: 65_536, stream: true, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA, effort: 'medium', + }); + await callJudge('score the whole bundle', 'claude-fable-5-1', { max_tokens: 65_536, stream: true, effort: 'medium' }); + await callJudge('score the whole bundle', 'claude-fable-5-1', { max_tokens: 65_536, stream: true }); + const bodies = transport.mock.calls.map(call => JSON.parse(call[1].body)); + expect(bodies[0].output_config).toEqual({ format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA }, effort: 'medium' }); + expect(bodies[1].output_config).toEqual({ effort: 'medium' }); + expect(bodies[2]).not.toHaveProperty('output_config'); + for (const body of bodies) expect(body).not.toHaveProperty('thinking'); +}); + test('the real SDK streams 64k requests and parses only completed public text', async () => { transport.mockResolvedValue(response()); expect(await callJudge('score the whole bundle', 'claude-fable-5-1', { diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 065c6e700..09bb8119b 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -434,6 +434,7 @@ async function runWorkflowJudge(opts: { structuredResponse?: boolean; maxTokens?: number; stream?: boolean; + effort?: 'medium'; model?: string; thresholds?: { clarity: number; completeness: number; actionability: number }; readInput?: () => WorkflowJudgeInput; @@ -520,7 +521,8 @@ async function runWorkflowJudge(opts: { try { result = reused?.samples ?? await judgePanel(() => callJudge(prompt, opts.model, { signal: controller.signal, max_tokens: maxTokens, ...(opts.stream ? { stream: true } : {}), - ...(opts.structuredResponse ? { jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } : {}) })); + ...(opts.structuredResponse ? { jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } : {}), + ...(opts.effort ? { effort: opts.effort } : {}) })); } catch (error) { checkActive(); if (error instanceof JudgeRefusalError && customInputMetadata) { @@ -572,6 +574,8 @@ describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'doc structuredResponse: true, maxTokens: 65_536, stream: true, + // Default effort thought past JUDGE_MS in 3 of 18 measured samples; medium kept all 18 under 80 s. + effort: 'medium', suite: 'Ship & Release skill evals', agentCapability: 'frontier', // The contract now precedes platform detection; keep the complete workflow. diff --git a/test/workflow-judge-cache.test.ts b/test/workflow-judge-cache.test.ts index 1d55fad33..5e230bbff 100644 --- a/test/workflow-judge-cache.test.ts +++ b/test/workflow-judge-cache.test.ts @@ -398,6 +398,8 @@ test('Ship sends its authorized 64k cap and compact response contract through th expect(options.structuredResponse).toBe(true); expect(options.maxTokens).toBe(65_536); expect(options.stream).toBe(true); + expect(options.effort).toBe('medium'); + expect(source.match(/^\s+effort: '/gm)).toHaveLength(1); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern'); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength'); expect(source.match(/structuredResponse: true/g)).toHaveLength(1); @@ -407,10 +409,10 @@ test('Ship sends its authorized 64k cap and compact response contract through th }) } as any); try { const h = actualCallback(f, { judge: (prompt, model, request) => callJudge(prompt, model, request) }); - await h.run({ ...h.options, structuredResponse: options.structuredResponse, maxTokens: options.maxTokens, stream: options.stream }); + await h.run({ ...h.options, structuredResponse: options.structuredResponse, maxTokens: options.maxTokens, stream: options.stream, effort: options.effort }); expect(stream.mock.calls[0]).toEqual([{ model: resolveEvalModel('judge'), max_tokens: 65_536, - output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } }, + output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA }, effort: 'medium' }, messages: [{ role: 'user', content: f.opts.prompt }], }, { signal: h.signals[0] }]); expect(h.records[0]).toMatchObject({ passed: true, judge_scores: { clarity: 4, completeness: 5, actionability: 4 } }); @@ -419,6 +421,8 @@ test('Ship sends its authorized 64k cap and compact response contract through th f.opts.structuredResponse = true; f.opts.maxTokens = 65_536; f.opts.stream = true; + expect(f.cache().lookup()).toBeNull(); + f.opts.effort = 'medium'; expect(f.cache().lookup()?.samples).toEqual(panel); } finally { stream.mockRestore(); } });