test: W2 judge calibration harness (W2 R6)

scripts/judge-calibration.ts draws two old 3-sample panels and one
schema-transport panel per corpus item, each sample a direct callJudge call
(never judgePanel or the workflow passing-panel cache, so old-vs-old flips
are real). Every sample records a value or an error kind (refusal,
truncation, parse, schema, transport), latency and priced usage via a new
callJudge onResponse observer. The manifest prices every call with the free
count_tokens API in priority order; dispatch stops at the budget or after two
consecutive transport failures. Analysis applies the eng landing rule (flips
<= old + 1, agreement does not drop, zero new false passes, p95 <= 80% of
JUDGE_MS), reports the held-out third separately, and flags the UC3 prose
trigger. Free stubbed tests cover error accounting, cache bypass, latency,
landing, stops and pricing.

scripts/judge-calibration.ts

@@ -0,0 +1,369 @@
+#!/usr/bin/env bun
+/**
+ * W2 judge calibration harness (comparison 1: schema transport, prose unchanged).
+ *
+ * For every corpus item of a request configuration it draws three independent
+ * 3-sample panels: two with the current request ("old-a", "old-b") and one with
+ * the schema-transport request ("new"). Each sample is one direct callJudge
+ * call. Calibration never goes through judgePanel or the workflow passing-panel
+ * cache: a replayed panel would make old-vs-old flips read zero.
+ *
+ * Each sample records a verdict-bearing value or an error kind (refusal,
+ * truncation, parse, schema, transport), its latency and its priced usage.
+ * Errors never count as verdicts; flip rates use items where both panels
+ * completed (all three samples valid).
+ *
+ * Usage:
+ *   bun run scripts/judge-calibration.ts manifest [--budget 90]
+ *   bun run scripts/judge-calibration.ts run --config <id> --budget <usd> [--concurrency 6]
+ *   bun run scripts/judge-calibration.ts analyze --config <id>
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { createHash } from 'node:crypto';
+import Anthropic from '@anthropic-ai/sdk';
+import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from '../test/helpers/llm-judge';
+import { JUDGE_MS } from '../test/helpers/eval-budgets';
+import { estimateCostUsd } from '../test/helpers/pricing';
+
+export const PANEL_SAMPLES = 3;
+export const PHASES = ['old-a', 'old-b', 'new'] as const;
+export type Phase = typeof PHASES[number];
+export type Verdict = 'pass' | 'fail';
+export type ErrorKind = 'refusal' | 'truncation' | 'parse' | 'schema' | 'transport';
+export const LATENCY_CEILING_MS = 0.8 * JUDGE_MS;
+export const CORPUS_ROOT = path.join(import.meta.dir, '..', 'test', 'fixtures', 'judge-calibration');
+
+export interface CorpusItem {
+  id: string;
+  category: 'pass' | 'fail' | 'adjacent';
+  expected: Verdict;
+  split: 'dev' | 'heldout';
+  rationale: string;
+  inputs: Record<string, string>;
+}
+
+export interface CalibrationConfig {
+  id: string;
+  /** Dispatch order when the priced manifest exceeds the budget (armJudge, inline judges, workflow by case count). */
+  priority: number;
+  /** Eval cases that send this request configuration. */
+  cases: string[];
+  model: string;
+  build(inputs: Record<string, string>): string;
+  /** Today's request options. */
+  oldOptions: CallJudgeOptions;
+  /** Comparison (1): today's options plus the JSON schema; prose unchanged. */
+  newOptions: CallJudgeOptions;
+  /** Throws when production would reject the sample's shape. */
+  validate(value: unknown): void;
+  verdict(samples: unknown[]): Verdict;
+  /** Manifest pricing estimate per call (output includes adaptive thinking). */
+  estimatedOutputTokens: number;
+}
+
+export interface SampleRecord {
+  config: string;
+  item: string;
+  split: CorpusItem['split'];
+  phase: Phase;
+  sample: number;
+  status: 'ok' | 'error';
+  error_kind?: ErrorKind;
+  error?: string;
+  value?: unknown;
+  latency_ms: number;
+  stop_reason: string | null;
+  usage: { input_tokens: number; output_tokens: number } | null;
+  cost_usd: number;
+}
+
+export type JudgeCall = (prompt: string, model: string, opts: CallJudgeOptions) => Promise<unknown>;
+
+export function loadCorpus(configId: string, root = CORPUS_ROOT): CorpusItem[] {
+  const dir = path.join(root, configId);
+  const corpus = JSON.parse(fs.readFileSync(path.join(dir, 'corpus.json'), 'utf8'));
+  return corpus.items.map((item: any) => ({
+    ...item,
+    inputs: Object.fromEntries(Object.entries(item.inputs as Record<string, string | { file: string }>).map(([key, value]) =>
+      [key, typeof value === 'string' ? value : fs.readFileSync(path.join(dir, value.file), 'utf8')])),
+  }));
+}
+
+export function classifyError(error: unknown): ErrorKind {
+  if (error instanceof JudgeRefusalError) return 'refusal';
+  const message = error instanceof Error ? error.message : String(error);
+  if (/truncated at max_tokens|stop_reason=max_tokens/.test(message)) return 'truncation';
+  if (error instanceof SyntaxError || /non-JSON|did not complete/.test(message)) return 'parse';
+  return 'transport';
+}
+
+/** The fixed request identity: what must not move between dispatch and analysis. */
+export function requestHash(config: CalibrationConfig, phase: Phase): string {
+  const { onResponse: _ignored, signal: _signal, ...options } = phase === 'new' ? config.newOptions : config.oldOptions;
+  return createHash('sha256').update(JSON.stringify({ model: config.model, options })).digest('hex');
+}
+
+export interface PlannedCall { item: CorpusItem; phase: Phase; sample: number }
+
+/** Every paid call for one configuration, interleaved by a fixed seed so phases share time-of-day drift. */
+export function planCalls(items: CorpusItem[], split: CorpusItem['split']): PlannedCall[] {
+  const calls = items.filter(item => item.split === split)
+    .flatMap(item => PHASES.flatMap(phase => Array.from({ length: PANEL_SAMPLES }, (_, sample) => ({ item, phase, sample }))));
+  let seed = 0x5eed;
+  const random = () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31;
+  for (let i = calls.length - 1; i > 0; i--) {
+    const j = Math.floor(random() * (i + 1));
+    [calls[i], calls[j]] = [calls[j]!, calls[i]!];
+  }
+  return calls;
+}
+
+export async function runSample(config: CalibrationConfig, planned: PlannedCall, call: JudgeCall = callJudge as JudgeCall,
+  clock: () => number = () => performance.now()): Promise<SampleRecord> {
+  let meta: JudgeResponseMeta | null = null;
+  const options = { ...(planned.phase === 'new' ? config.newOptions : config.oldOptions), onResponse: (m: JudgeResponseMeta) => { meta = m; } };
+  const base = { config: config.id, item: planned.item.id, split: planned.item.split, phase: planned.phase, sample: planned.sample };
+  const started = clock();
+  let value: unknown;
+  let failure: { kind: ErrorKind; message: string } | null = null;
+  try {
+    value = await call(config.build(planned.item.inputs), config.model, options);
+    try { config.validate(value); }
+    catch (error) { failure = { kind: 'schema', message: error instanceof Error ? error.message : String(error) }; }
+  } catch (error) {
+    failure = { kind: classifyError(error), message: error instanceof Error ? error.message : String(error) };
+  }
+  const latency_ms = Math.round(clock() - started);
+  const response = meta as JudgeResponseMeta | null;
+  const usage = response?.usage ?? null;
+  const cost_usd = usage ? estimateCostUsd({ input: usage.input_tokens, output: usage.output_tokens }, config.model) : 0;
+  const shared = { latency_ms, stop_reason: response?.stop_reason ?? null, usage, cost_usd };
+  return failure
+    ? { ...base, status: 'error', error_kind: failure.kind, error: failure.message.slice(0, 500), ...shared }
+    : { ...base, status: 'ok', value, ...shared };
+}
+
+export interface RunOptions {
+  budgetUsd: number;
+  concurrency?: number;
+  call?: JudgeCall;
+  clock?: () => number;
+  onRecord?: (record: SampleRecord) => void;
+}
+
+/**
+ * Dispatch planned calls with bounded concurrency. Stops dispatching when the
+ * actual spend plus one in-flight estimate per open slot would pass the budget,
+ * or after two consecutive transport failures (infrastructure, not a verdict).
+ */
+export async function runCalls(config: CalibrationConfig, calls: PlannedCall[], opts: RunOptions): Promise<{ records: SampleRecord[]; stopped: string | null }> {
+  const records: SampleRecord[] = [];
+  const perCall = estimateCostUsd({ input: 0, output: config.estimatedOutputTokens }, config.model);
+  let spent = 0, next = 0, consecutiveTransport = 0, inFlight = 0;
+  let stopped: string | null = null;
+  const worker = async () => {
+    while (!stopped && next < calls.length) {
+      if (spent + (inFlight + 1) * perCall * 2 > opts.budgetUsd) { stopped = `budget: spent $${spent.toFixed(2)} of $${opts.budgetUsd.toFixed(2)}`; break; }
+      const planned = calls[next++]!;
+      inFlight++;
+      const record = await runSample(config, planned, opts.call, opts.clock);
+      inFlight--;
+      spent += record.cost_usd;
+      consecutiveTransport = record.error_kind === 'transport' ? consecutiveTransport + 1 : 0;
+      records.push(record);
+      opts.onRecord?.(record);
+      if (consecutiveTransport >= 2) stopped = `infrastructure: two consecutive transport failures (${record.error})`;
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, opts.concurrency ?? 6) }, worker));
+  return { records, stopped };
+}
+
+export interface PanelResult { verdict: Verdict | null; errors: number }
+
+export function panels(config: CalibrationConfig, records: SampleRecord[]): Map<string, Record<Phase, PanelResult>> {
+  const byItem = new Map<string, Record<Phase, PanelResult>>();
+  for (const item of new Set(records.map(record => record.item))) {
+    const entry = {} as Record<Phase, PanelResult>;
+    for (const phase of PHASES) {
+      const samples = records.filter(record => record.item === item && record.phase === phase);
+      const ok = samples.filter(record => record.status === 'ok');
+      entry[phase] = { verdict: ok.length === PANEL_SAMPLES && samples.length === PANEL_SAMPLES ? config.verdict(ok.map(record => record.value)) : null,
+        errors: samples.length - ok.length };
+    }
+    byItem.set(item, entry);
+  }
+  return byItem;
+}
+
+function p95(values: number[]): number | null {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.ceil(0.95 * sorted.length) - 1)]!;
+}
+
+export interface ComparisonSummary {
+  items: number;
+  old_flips: { flips: number; compared: number };
+  new_flips: { flips: number; compared: number };
+  agreement: Record<Phase, number>;
+  false_passes: Record<Phase, string[]>;
+  new_false_passes: string[];
+}
+
+export function compare(config: CalibrationConfig, items: CorpusItem[], records: SampleRecord[]): ComparisonSummary {
+  const byItem = panels(config, records);
+  const flips = (a: Phase, b: Phase) => {
+    const both = items.map(item => byItem.get(item.id)).filter(entry => entry?.[a].verdict && entry[b].verdict);
+    return { flips: both.filter(entry => entry![a].verdict !== entry![b].verdict).length, compared: both.length };
+  };
+  const agreement = Object.fromEntries(PHASES.map(phase =>
+    [phase, items.filter(item => byItem.get(item.id)?.[phase].verdict === item.expected).length])) as Record<Phase, number>;
+  const falsePasses = Object.fromEntries(PHASES.map(phase =>
+    [phase, items.filter(item => item.expected === 'fail' && byItem.get(item.id)?.[phase].verdict === 'pass').map(item => item.id)])) as Record<Phase, string[]>;
+  return { items: items.length, old_flips: flips('old-a', 'old-b'), new_flips: flips('new', 'old-a'), agreement,
+    false_passes: falsePasses, new_false_passes: falsePasses.new.filter(id => !falsePasses['old-a'].includes(id)) };
+}
+
+export interface CalibrationResult {
+  config: string;
+  model: string;
+  request_hash: Record<'old' | 'new', string>;
+  complete: boolean;
+  stopped: string | null;
+  calls: number;
+  cost_usd: number;
+  errors: Record<'old' | 'new', Record<ErrorKind, number> & { samples: number; rate: number }>;
+  latency_p95_ms: Record<'old' | 'new', number | null>;
+  all: ComparisonSummary;
+  heldout: ComparisonSummary;
+  landing: { lands: boolean; reasons: string[]; strict_no_flip_increase: boolean };
+  prose_step_triggered: { triggered: boolean; reasons: string[] };
+}
+
+export function analyze(config: CalibrationConfig, items: CorpusItem[], records: SampleRecord[], stopped: string | null = null): CalibrationResult {
+  const prompt = (phase: Phase) => phase === 'new' ? 'new' : 'old';
+  const errors = Object.fromEntries((['old', 'new'] as const).map(which => {
+    const samples = records.filter(record => prompt(record.phase) === which);
+    const counts = Object.fromEntries((['refusal', 'truncation', 'parse', 'schema', 'transport'] as const)
+      .map(kind => [kind, samples.filter(record => record.error_kind === kind).length])) as Record<ErrorKind, number>;
+    const errored = samples.filter(record => record.status === 'error').length;
+    return [which, { ...counts, samples: samples.length, rate: samples.length ? errored / samples.length : 0 }];
+  })) as CalibrationResult['errors'];
+  const latency = Object.fromEntries((['old', 'new'] as const).map(which =>
+    [which, p95(records.filter(record => prompt(record.phase) === which && record.status === 'ok').map(record => record.latency_ms))])) as CalibrationResult['latency_p95_ms'];
+  const expectedCalls = items.length * PHASES.length * PANEL_SAMPLES;
+  const complete = records.length === expectedCalls && !stopped;
+  const all = compare(config, items, records);
+  const heldout = compare(config, items.filter(item => item.split === 'heldout'), records);
+
+  const reasons: string[] = [];
+  if (!complete) reasons.push(`incomplete: ${records.length}/${expectedCalls} calls${stopped ? ` (${stopped})` : ''}`);
+  if (all.new_flips.flips > all.old_flips.flips + 1) reasons.push(`new flips ${all.new_flips.flips}/${all.new_flips.compared} > old flips ${all.old_flips.flips}/${all.old_flips.compared} + 1`);
+  if (all.agreement.new < all.agreement['old-a']) reasons.push(`expected-verdict agreement dropped: new ${all.agreement.new} < old ${all.agreement['old-a']}`);
+  if (all.new_false_passes.length > 0) reasons.push(`new false passes: ${all.new_false_passes.join(', ')}`);
+  if (latency.new !== null && latency.new > LATENCY_CEILING_MS) reasons.push(`new p95 latency ${latency.new} ms > ${LATENCY_CEILING_MS} ms (80% of JUDGE_MS)`);
+
+  const trigger: string[] = [];
+  if (records.some(record => record.phase === 'new' && record.status === 'error')) trigger.push('new prompt has errored samples');
+  if (reasons.length > 0) trigger.push('a landing condition failed');
+  if (all.old_flips.flips >= 3) trigger.push(`old-vs-old flips ${all.old_flips.flips} >= 3`);
+
+  return {
+    config: config.id, model: config.model,
+    request_hash: { old: requestHash(config, 'old-a'), new: requestHash(config, 'new') },
+    complete, stopped, calls: records.length,
+    cost_usd: +records.reduce((sum, record) => sum + record.cost_usd, 0).toFixed(4),
+    errors, latency_p95_ms: latency, all, heldout,
+    landing: { lands: reasons.length === 0, reasons, strict_no_flip_increase: all.new_flips.flips <= all.old_flips.flips },
+    prose_step_triggered: { triggered: trigger.length > 0, reasons: trigger },
+  };
+}
+
+export interface ManifestEntry {
+  config: string;
+  priority: number;
+  cases: number;
+  model: string;
+  items: number;
+  calls: number;
+  input_tokens: number;
+  estimated_output_tokens_per_call: number;
+  estimated_cost_usd: number;
+}
+
+/** Price every planned call before dispatch; count_tokens is free. */
+export async function priceManifest(configs: CalibrationConfig[], countTokens: (model: string, prompt: string) => Promise<number>,
+  root = CORPUS_ROOT): Promise<ManifestEntry[]> {
+  const entries: ManifestEntry[] = [];
+  for (const config of [...configs].sort((a, b) => a.priority - b.priority)) {
+    if (!fs.existsSync(path.join(root, config.id, 'corpus.json'))) continue;
+    const items = loadCorpus(config.id, root);
+    let input = 0;
+    for (const item of items) input += await countTokens(config.model, config.build(item.inputs)) * PHASES.length * PANEL_SAMPLES;
+    const calls = items.length * PHASES.length * PANEL_SAMPLES;
+    entries.push({ config: config.id, priority: config.priority, cases: config.cases.length, model: config.model, items: items.length, calls,
+      input_tokens: input, estimated_output_tokens_per_call: config.estimatedOutputTokens,
+      estimated_cost_usd: estimateCostUsd({ input, output: calls * config.estimatedOutputTokens }, config.model) });
+  }
+  return entries;
+}
+
+function flag(args: string[], name: string): string | undefined {
+  const index = args.indexOf(`--${name}`);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+async function main(args: string[]): Promise<void> {
+  // The per-judge table (builders, schemas, verdict rules) is committed separately from this harness.
+  const { CALIBRATION_CONFIGS } = await import(path.join(import.meta.dir, 'judge-calibration-configs.ts')) as { CALIBRATION_CONFIGS: CalibrationConfig[] };
+  const command = args[0];
+  if (command === 'manifest') {
+    const client = new Anthropic();
+    const entries = await priceManifest(CALIBRATION_CONFIGS, async (model, prompt) =>
+      (await client.messages.countTokens({ model, messages: [{ role: 'user', content: prompt }] })).input_tokens);
+    const budget = Number(flag(args, 'budget') ?? 90);
+    let running = 0;
+    const manifest = { generated_at: new Date().toISOString(), budget_usd: budget,
+      entries: entries.map(entry => ({ ...entry, cumulative_usd: +(running += entry.estimated_cost_usd).toFixed(4), within_budget: running <= budget })) };
+    fs.writeFileSync(path.join(CORPUS_ROOT, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+    console.table(manifest.entries);
+    return;
+  }
+  const config = CALIBRATION_CONFIGS.find(candidate => candidate.id === flag(args, 'config'));
+  if (!config) throw new Error(`unknown --config; expected one of ${CALIBRATION_CONFIGS.map(candidate => candidate.id).join(', ')}`);
+  const items = loadCorpus(config.id);
+  const runsDir = path.join(CORPUS_ROOT, config.id, 'runs');
+  const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl');
+  const resultPath = path.join(runsDir, 'comparison-1.result.json');
+  if (command === 'run') {
+    if (fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`);
+    fs.mkdirSync(runsDir, { recursive: true });
+    const budgetUsd = Number(flag(args, 'budget'));
+    if (!Number.isFinite(budgetUsd) || budgetUsd <= 0) throw new Error('--budget <usd> is required');
+    const concurrency = Number(flag(args, 'concurrency') ?? 6);
+    const onRecord = (record: SampleRecord) => fs.appendFileSync(samplesPath, JSON.stringify(record) + '\n');
+    let spent = 0, stopped: string | null = null;
+    // Development items first; the held-out third is dispatched only after, with the same frozen requests.
+    for (const split of ['dev', 'heldout'] as const) {
+      const outcome = await runCalls(config, planCalls(items, split), { budgetUsd: budgetUsd - spent, concurrency, onRecord });
+      spent += outcome.records.reduce((sum, record) => sum + record.cost_usd, 0);
+      console.log(`[calibration] ${config.id} ${split}: ${outcome.records.length} calls, $${spent.toFixed(2)} cumulative${outcome.stopped ? `, stopped: ${outcome.stopped}` : ''}`);
+      if ((stopped = outcome.stopped)) break;
+    }
+    fs.writeFileSync(path.join(runsDir, 'comparison-1.stop.json'), JSON.stringify({ stopped }) + '\n');
+  } else if (command !== 'analyze') {
+    throw new Error(`unknown command ${command}; expected manifest, run or analyze`);
+  }
+  const records: SampleRecord[] = fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line));
+  const stopPath = path.join(runsDir, 'comparison-1.stop.json');
+  const stopped = fs.existsSync(stopPath) ? JSON.parse(fs.readFileSync(stopPath, 'utf8')).stopped : null;
+  const result = analyze(config, items, records, stopped);
+  fs.writeFileSync(resultPath, JSON.stringify(result, null, 2) + '\n');
+  console.log(JSON.stringify(result, null, 2));
+}
+
+if (import.meta.main) {
+  main(process.argv.slice(2)).catch(error => { console.error(error); process.exit(1); });
+}

test/helpers/llm-judge.ts

@@ -106,6 +106,15 @@ export interface CallJudgeOptions {
   jsonSchema?: JSONOutputFormat['schema'];
   /** Adaptive-thinking effort; the judge models accept no thinking token budget. */
   effort?: 'low' | 'medium' | 'high';
+  /** Observes the provider response before parsing (calibration cost and stop accounting); never sent. */
+  onResponse?: (response: JudgeResponseMeta) => void;
+}
+
+export interface JudgeResponseMeta {
+  id: string | null;
+  model: string | null;
+  stop_reason: string | null;
+  usage: { input_tokens: number; output_tokens: number } | null;
 }
 
 export async function callJudge<T>(
@@ -166,6 +175,13 @@ export async function callJudge<T>(
     }
   }
 
+  opts?.onResponse?.({
+    id: typeof response.id === 'string' ? response.id : null,
+    model: typeof response.model === 'string' ? response.model : null,
+    stop_reason: response.stop_reason ?? null,
+    usage: typeof response.usage?.input_tokens === 'number' && typeof response.usage?.output_tokens === 'number'
+      ? { input_tokens: response.usage.input_tokens, output_tokens: response.usage.output_tokens } : null,
+  });
   const text = response.content
     .filter(block => block.type === 'text')
     .map(block => block.text)

test/judge-calibration.test.ts

@@ -0,0 +1,186 @@
+import { describe, expect, spyOn, test } from 'bun:test';
+import Anthropic from '@anthropic-ai/sdk';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge';
+import {
+  analyze, classifyError, loadCorpus, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
+  type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord,
+} from '../scripts/judge-calibration';
+
+const item = (id: string, expected: 'pass' | 'fail', split: 'dev' | 'heldout' = 'dev'): CorpusItem =>
+  ({ id, category: expected, expected, split, rationale: 'fixture', inputs: { text: `doc ${id}` } });
+
+const config: CalibrationConfig = {
+  id: 'stub', priority: 1, cases: ['a', 'b'], model: 'claude-fable-5-1',
+  build: inputs => `grade: ${inputs.text}`,
+  oldOptions: {},
+  newOptions: { jsonSchema: { type: 'object' } },
+  validate: value => { if (typeof (value as { score?: unknown })?.score !== 'number') throw new Error('score missing'); },
+  verdict: samples => samples.filter(sample => (sample as { score: number }).score >= 4).length * 2 > samples.length ? 'pass' : 'fail',
+  estimatedOutputTokens: 1000,
+};
+
+/** Stub provider keyed by prompt and phase; usage is reported through the real onResponse seam. */
+function stub(score: (prompt: string, opts: CallJudgeOptions, n: number) => unknown): { call: JudgeCall; calls: Array<{ prompt: string; opts: CallJudgeOptions }> } {
+  const calls: Array<{ prompt: string; opts: CallJudgeOptions }> = [];
+  return {
+    calls,
+    call: async (prompt, _model, opts) => {
+      calls.push({ prompt, opts });
+      opts.onResponse?.({ id: 'msg', model: 'claude-fable-5-1', stop_reason: 'end_turn', usage: { input_tokens: 1000, output_tokens: 1000 } });
+      const value = score(prompt, opts, calls.length);
+      if (value instanceof Error) throw value;
+      return value;
+    },
+  };
+}
+
+const records = async (items: CorpusItem[], call: JudgeCall) => {
+  const out: SampleRecord[] = [];
+  for (const split of ['dev', 'heldout'] as const) out.push(...(await runCalls(config, planCalls(items, split), { budgetUsd: 1000, call })).records);
+  return out;
+};
+
+describe('judge calibration harness (free, stubbed provider)', () => {
+  test('every item gets two independent old panels and one new panel, each sample a direct call', async () => {
+    const items = [item('p1', 'pass'), item('f1', 'fail'), item('h1', 'pass', 'heldout')];
+    const provider = stub(() => ({ score: 5 }));
+    const out = await records(items, provider.call);
+    expect(provider.calls).toHaveLength(items.length * PHASES.length * PANEL_SAMPLES);
+    for (const entry of items) {
+      for (const phase of PHASES) expect(out.filter(record => record.item === entry.id && record.phase === phase)).toHaveLength(PANEL_SAMPLES);
+    }
+    expect(provider.calls.filter(call => call.opts.jsonSchema)).toHaveLength(items.length * PANEL_SAMPLES);
+    expect(new Set(provider.calls.map(call => call.prompt))).toEqual(new Set(items.map(entry => `grade: ${entry.inputs.text}`)));
+    // Held-out samples are dispatched only after every development sample.
+    const firstHeldout = out.findIndex(record => record.split === 'heldout');
+    expect(out.slice(firstHeldout).every(record => record.split === 'heldout')).toBe(true);
+  });
+
+  test('the harness never routes through judgePanel or the workflow passing-panel cache', () => {
+    const scripts = path.join(import.meta.dir, '..', 'scripts');
+    for (const file of ['judge-calibration.ts', 'judge-calibration-configs.ts'].filter(name => fs.existsSync(path.join(scripts, name)))) {
+      const source = fs.readFileSync(path.join(scripts, file), 'utf8');
+      const imports = source.match(/^import[\s\S]*?from '[^']+';$/gm) ?? [];
+      expect(imports.length).toBeGreaterThan(0);
+      expect(imports.join('\n')).not.toMatch(/\bjudgePanel\b|workflow-judge-cache|eval-input-cache/);
+      expect(source.replace(/^\s*(?:\/\*\*|\*|\/\/).*$/gm, '')).not.toMatch(/\bjudgePanel\(|prepareWorkflowJudgeCache|lookupEvalInputCache/);
+    }
+  });
+
+  test('a second old panel is sampled, not replayed: old-vs-old flips are counted', async () => {
+    const items = [item('p1', 'pass')];
+    const provider = stub((_prompt, opts, n) => ({ score: opts.jsonSchema ? 5 : n <= 3 ? 5 : 2 }));
+    // Force the old panels apart: the first three old calls pass, the next three fail.
+    let oldCalls = 0;
+    const call: JudgeCall = (prompt, model, opts) => provider.call(prompt, model, opts).then(value =>
+      opts.jsonSchema ? value : ({ score: ++oldCalls <= 3 ? 5 : 2 }));
+    const result = analyze(config, items, await records(items, call));
+    expect(result.all.old_flips).toEqual({ flips: 1, compared: 1 });
+  });
+
+  test('error kinds are recorded per sample and never count as verdicts', async () => {
+    expect(classifyError(new JudgeRefusalError({ content: [] }))).toBe('refusal');
+    expect(classifyError(new Error('Judge response truncated at max_tokens=8192 (model=m)'))).toBe('truncation');
+    expect(classifyError(new SyntaxError('Unexpected token'))).toBe('parse');
+    expect(classifyError(new Error('Judge returned non-JSON: hi'))).toBe('parse');
+    expect(classifyError(new Error('Structured judge did not complete: stop_reason=pause_turn'))).toBe('parse');
+    expect(classifyError(Object.assign(new Error('Overloaded'), { status: 529 }))).toBe('transport');
+
+    const planned = { item: item('p1', 'pass'), phase: 'new' as const, sample: 0 };
+    const schema = await runSample(config, planned, stub(() => ({ verdict: 'yes' })).call);
+    expect(schema).toMatchObject({ status: 'error', error_kind: 'schema', cost_usd: 0.06 });
+    expect(schema).not.toHaveProperty('value');
+    const refusal = await runSample(config, planned, stub(() => new JudgeRefusalError({ content: [] })).call);
+    expect(refusal).toMatchObject({ status: 'error', error_kind: 'refusal' });
+
+    // One errored new sample leaves that panel incomplete: excluded from flips, counted as an error.
+    const items = [item('p1', 'pass'), item('p2', 'pass')];
+    let newCalls = 0;
+    const result = analyze(config, items, await records(items, stub((_prompt, opts) =>
+      opts.jsonSchema && ++newCalls === 1 ? new SyntaxError('bad json') : { score: 5 }).call));
+    expect(result.errors.new).toMatchObject({ parse: 1, samples: 6 });
+    expect(result.errors.old).toMatchObject({ parse: 0, samples: 12, rate: 0 });
+    expect(result.all.new_flips.compared).toBe(1);
+    expect(result.all.old_flips.compared).toBe(2);
+    expect(result.prose_step_triggered.triggered).toBe(true);
+    expect(result.prose_step_triggered.reasons).toContain('new prompt has errored samples');
+  });
+
+  test('latency is measured per sample and the p95 gate holds the schema back', async () => {
+    let now = 0;
+    const clock = () => now;
+    const slow: JudgeCall = async (_prompt, _model, opts) => { now += opts.jsonSchema ? 100_000 : 10_000; return { score: 5 }; };
+    const items = [item('p1', 'pass')];
+    const out: SampleRecord[] = (await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 1000, call: slow, clock, concurrency: 1 })).records;
+    expect(out.filter(record => record.phase === 'new').every(record => record.latency_ms === 100_000)).toBe(true);
+    const result = analyze(config, items, out);
+    expect(result.latency_p95_ms).toEqual({ old: 10_000, new: 100_000 });
+    expect(result.landing.lands).toBe(false);
+    expect(result.landing.reasons.join(' ')).toContain('p95 latency');
+  });
+
+  test('landing: flips at most old+1, agreement does not drop, zero new false passes', async () => {
+    const items = [item('p1', 'pass'), item('f1', 'fail'), item('f2', 'fail')];
+    const same = analyze(config, items, await records(items, stub(prompt => ({ score: prompt.includes('p1') ? 5 : 1 })).call));
+    expect(same.landing).toEqual({ lands: true, reasons: [], strict_no_flip_increase: true });
+    expect(same.all.agreement).toEqual({ 'old-a': 3, 'old-b': 3, new: 3 });
+
+    const leaky = analyze(config, items, await records(items, stub((prompt, opts) =>
+      ({ score: prompt.includes('p1') || (opts.jsonSchema && prompt.includes('f1')) ? 5 : 1 })).call));
+    expect(leaky.all.new_false_passes).toEqual(['f1']);
+    expect(leaky.landing.lands).toBe(false);
+    expect(leaky.landing.reasons.join(' ')).toContain('new false passes: f1');
+    expect(leaky.landing.reasons.join(' ')).toContain('agreement dropped');
+  });
+
+  test('two consecutive transport failures stop the study; the budget stops dispatch before it is passed', async () => {
+    const items = [item('p1', 'pass'), item('p2', 'pass')];
+    const down = await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 1000, concurrency: 1,
+      call: stub(() => Object.assign(new Error('socket hang up'), { status: 500 })).call });
+    expect(down.records).toHaveLength(2);
+    expect(down.stopped).toContain('infrastructure');
+
+    const capped = await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 0.5, concurrency: 1, call: stub(() => ({ score: 5 })).call });
+    expect(capped.stopped).toContain('budget');
+    expect(capped.records.reduce((sum, record) => sum + record.cost_usd, 0)).toBeLessThanOrEqual(0.5);
+    const result = analyze(config, items, capped.records, capped.stopped);
+    expect(result.complete).toBe(false);
+    expect(result.landing.lands).toBe(false);
+  });
+
+  test('callJudge reports usage and stop reason through onResponse without sending it', async () => {
+    const key = process.env.ANTHROPIC_API_KEY;
+    process.env.ANTHROPIC_API_KEY = 'test-only-key';
+    const create = spyOn(Anthropic.Messages.prototype, 'create').mockResolvedValue({ id: 'msg_1', model: 'claude-fable-5-1',
+      stop_reason: 'end_turn', content: [{ type: 'text', text: '{"score":4}' }], usage: { input_tokens: 12, output_tokens: 34 } } as never);
+    try {
+      const seen: JudgeResponseMeta[] = [];
+      expect(await callJudge<{ score: number }>('grade', 'claude-fable-5-1', { onResponse: meta => seen.push(meta) })).toEqual({ score: 4 });
+      expect(seen).toEqual([{ id: 'msg_1', model: 'claude-fable-5-1', stop_reason: 'end_turn', usage: { input_tokens: 12, output_tokens: 34 } }]);
+      expect(create.mock.calls[0]![0]).toEqual({ model: 'claude-fable-5-1', max_tokens: 8192, messages: [{ role: 'user', content: 'grade' }] });
+    } finally {
+      create.mockRestore();
+      if (key === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = key;
+    }
+  });
+
+  test('the manifest prices every call in priority order from a committed corpus', async () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'judge-cal-'));
+    try {
+      for (const id of ['second', 'first']) {
+        fs.mkdirSync(path.join(root, id, 'inputs'), { recursive: true });
+        fs.writeFileSync(path.join(root, id, 'inputs', 'p1.text.md'), 'from file');
+        fs.writeFileSync(path.join(root, id, 'corpus.json'), JSON.stringify({ items: [
+          { ...item('p1', 'pass'), inputs: { text: { file: 'inputs/p1.text.md' } } }, item('f1', 'fail', 'heldout')] }));
+      }
+      expect(loadCorpus('first', root)[0]!.inputs.text).toBe('from file');
+      const entries = await priceManifest([{ ...config, id: 'second', priority: 2 }, { ...config, id: 'first', priority: 1 }, { ...config, id: 'absent', priority: 0 }],
+        async () => 2000, root);
+      expect(entries.map(entry => entry.config)).toEqual(['first', 'second']);
+      expect(entries[0]).toMatchObject({ items: 2, calls: 18, input_tokens: 36_000, estimated_cost_usd: 1.26 });
+    } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  });
+});