test(llm-judge): run the ship workflow judge at medium effort so its panel fits JUDGE_MS

claude-fable-5-1 accepts only adaptive thinking (thinking.type.enabled with
budget_tokens returns 400), so effort is the available thinking control.
Measured on the exact ship judge request (105,301 input tokens):

- default effort, 18 samples: thinking 5,086-10,881 tokens, 75.9-144.7 s;
  3 of 18 passed the 120 s deadline (about 42% of 3-sample panels).
- medium effort, 18 samples: thinking 2,749-5,762, output at most 6,144
  tokens, 43.1-77.9 s; scores 4/4/4 in 16 of 18 (clarity 3 in two), versus
  14 of 18 at default.

callJudge gains an effort option sent as output_config.effort; only the ship
judge sets it. Rubric, floors, panel size, deadline, model and max_tokens are
unchanged. The cache identity records effort.

test/helpers/llm-judge.ts

@@ -104,6 +104,8 @@ export interface CallJudgeOptions {
   signal?: AbortSignal;
   /** Opt-in serialization contract; callers still validate the judgment locally. */
   jsonSchema?: JSONOutputFormat['schema'];
+  /** Adaptive-thinking effort; the judge models accept no thinking token budget. */
+  effort?: 'low' | 'medium' | 'high';
 }
 
 export async function callJudge<T>(
@@ -129,7 +131,9 @@ export async function callJudge<T>(
     model: resolvedModel,
     max_tokens: maxTokens,
     ...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}),
-    ...(opts?.jsonSchema === undefined ? {} : { output_config: { format: { type: 'json_schema' as const, schema: opts.jsonSchema } } }),
+    ...(opts?.jsonSchema === undefined && opts?.effort === undefined ? {} : { output_config: {
+      ...(opts?.jsonSchema === undefined ? {} : { format: { type: 'json_schema' as const, schema: opts.jsonSchema } }),
+      ...(opts?.effort === undefined ? {} : { effort: opts.effort }) } }),
     messages: [{ role: 'user' as const, content: prompt }],
   };
   const makeRequest = () => opts?.stream

test/helpers/workflow-judge-cache.ts

@@ -18,6 +18,7 @@ export interface WorkflowCacheOptions {
   structuredResponse?: boolean;
   maxTokens?: number;
   stream?: boolean;
+  effort?: 'medium';
   env?: NodeJS.ProcessEnv;
 }
 export interface WorkflowJudgeReuse {
@@ -88,6 +89,7 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): {
           request: opts.stream ? 'messages.stream/user' : 'messages.create/user', retries: 0,
           panel: { samples: JUDGE_PANEL_SAMPLES, numeric: 'mean', boolean: 'majority' },
           ...(opts.stream ? { stream: true } : {}),
+          ...(opts.effort ? { effort: opts.effort } : {}),
           ...(opts.structuredResponse ? { output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } },
             response_validation: { reasoning_words_below: WORKFLOW_JUDGE_REASONING_WORD_LIMIT } } : {}) },
         runtime: { image: env.EVALS_CACHE_RUNTIME_ID!, bun: Bun.version, node: process.versions.node,

test/llm-judge-stream.test.ts

@@ -45,6 +45,20 @@ test('the pinned SDK refuses a default nonstreaming 64k request before network a
   expect(transport).not.toHaveBeenCalled();
 });
 
+test('effort reaches the wire beside the response schema and is absent unless requested', async () => {
+  transport.mockImplementation(async () => response());
+  await callJudge('score the whole bundle', 'claude-fable-5-1', {
+    max_tokens: 65_536, stream: true, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA, effort: 'medium',
+  });
+  await callJudge('score the whole bundle', 'claude-fable-5-1', { max_tokens: 65_536, stream: true, effort: 'medium' });
+  await callJudge('score the whole bundle', 'claude-fable-5-1', { max_tokens: 65_536, stream: true });
+  const bodies = transport.mock.calls.map(call => JSON.parse(call[1].body));
+  expect(bodies[0].output_config).toEqual({ format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA }, effort: 'medium' });
+  expect(bodies[1].output_config).toEqual({ effort: 'medium' });
+  expect(bodies[2]).not.toHaveProperty('output_config');
+  for (const body of bodies) expect(body).not.toHaveProperty('thinking');
+});
+
 test('the real SDK streams 64k requests and parses only completed public text', async () => {
   transport.mockResolvedValue(response());
   expect(await callJudge('score the whole bundle', 'claude-fable-5-1', {

test/skill-llm-eval.test.ts

@@ -434,6 +434,7 @@ async function runWorkflowJudge(opts: {
   structuredResponse?: boolean;
   maxTokens?: number;
   stream?: boolean;
+  effort?: 'medium';
   model?: string;
   thresholds?: { clarity: number; completeness: number; actionability: number };
   readInput?: () => WorkflowJudgeInput;
@@ -520,7 +521,8 @@ async function runWorkflowJudge(opts: {
     try {
       result = reused?.samples ?? await judgePanel(() => callJudge<JudgeScore>(prompt, opts.model, { signal: controller.signal, max_tokens: maxTokens,
         ...(opts.stream ? { stream: true } : {}),
-        ...(opts.structuredResponse ? { jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } : {}) }));
+        ...(opts.structuredResponse ? { jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } : {}),
+        ...(opts.effort ? { effort: opts.effort } : {}) }));
     } catch (error) {
       checkActive();
       if (error instanceof JudgeRefusalError && customInputMetadata) {
@@ -572,6 +574,8 @@ describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'doc
       structuredResponse: true,
       maxTokens: 65_536,
       stream: true,
+      // Default effort thought past JUDGE_MS in 3 of 18 measured samples; medium kept all 18 under 80 s.
+      effort: 'medium',
       suite: 'Ship & Release skill evals',
       agentCapability: 'frontier',
       // The contract now precedes platform detection; keep the complete workflow.

test/workflow-judge-cache.test.ts

@@ -398,6 +398,8 @@ test('Ship sends its authorized 64k cap and compact response contract through th
   expect(options.structuredResponse).toBe(true);
   expect(options.maxTokens).toBe(65_536);
   expect(options.stream).toBe(true);
+  expect(options.effort).toBe('medium');
+  expect(source.match(/^\s+effort: '/gm)).toHaveLength(1);
   expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern');
   expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength');
   expect(source.match(/structuredResponse: true/g)).toHaveLength(1);
@@ -407,10 +409,10 @@ test('Ship sends its authorized 64k cap and compact response contract through th
   }) } as any);
   try {
     const h = actualCallback(f, { judge: (prompt, model, request) => callJudge<typeof scores>(prompt, model, request) });
-    await h.run({ ...h.options, structuredResponse: options.structuredResponse, maxTokens: options.maxTokens, stream: options.stream });
+    await h.run({ ...h.options, structuredResponse: options.structuredResponse, maxTokens: options.maxTokens, stream: options.stream, effort: options.effort });
     expect(stream.mock.calls[0]).toEqual([{
       model: resolveEvalModel('judge'), max_tokens: 65_536,
-      output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } },
+      output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA }, effort: 'medium' },
       messages: [{ role: 'user', content: f.opts.prompt }],
     }, { signal: h.signals[0] }]);
     expect(h.records[0]).toMatchObject({ passed: true, judge_scores: { clarity: 4, completeness: 5, actionability: 4 } });
@@ -419,6 +421,8 @@ test('Ship sends its authorized 64k cap and compact response contract through th
     f.opts.structuredResponse = true;
     f.opts.maxTokens = 65_536;
     f.opts.stream = true;
+    expect(f.cache().lookup()).toBeNull();
+    f.opts.effort = 'medium';
     expect(f.cache().lookup()?.samples).toEqual(panel);
   } finally { stream.mockRestore(); }
 });