From 6185205bab836cdb987c0da1bf47adb9921c0916 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 20:38:26 +0000 Subject: [PATCH] test: W2 judge calibration harness (W2 R6) scripts/judge-calibration.ts draws two old 3-sample panels and one schema-transport panel per corpus item, each sample a direct callJudge call (never judgePanel or the workflow passing-panel cache, so old-vs-old flips are real). Every sample records a value or an error kind (refusal, truncation, parse, schema, transport), latency and priced usage via a new callJudge onResponse observer. The manifest prices every call with the free count_tokens API in priority order; dispatch stops at the budget or after two consecutive transport failures. Analysis applies the eng landing rule (flips <= old + 1, agreement does not drop, zero new false passes, p95 <= 80% of JUDGE_MS), reports the held-out third separately, and flags the UC3 prose trigger. Free stubbed tests cover error accounting, cache bypass, latency, landing, stops and pricing. --- scripts/judge-calibration.ts | 369 +++++++++++++++++++++++++++++++++ test/helpers/llm-judge.ts | 16 ++ test/judge-calibration.test.ts | 186 +++++++++++++++++ 3 files changed, 571 insertions(+) create mode 100644 scripts/judge-calibration.ts create mode 100644 test/judge-calibration.test.ts diff --git a/scripts/judge-calibration.ts b/scripts/judge-calibration.ts new file mode 100644 index 000000000..68e8fae95 --- /dev/null +++ b/scripts/judge-calibration.ts @@ -0,0 +1,369 @@ +#!/usr/bin/env bun +/** + * W2 judge calibration harness (comparison 1: schema transport, prose unchanged). + * + * For every corpus item of a request configuration it draws three independent + * 3-sample panels: two with the current request ("old-a", "old-b") and one with + * the schema-transport request ("new"). Each sample is one direct callJudge + * call. Calibration never goes through judgePanel or the workflow passing-panel + * cache: a replayed panel would make old-vs-old flips read zero. + * + * Each sample records a verdict-bearing value or an error kind (refusal, + * truncation, parse, schema, transport), its latency and its priced usage. + * Errors never count as verdicts; flip rates use items where both panels + * completed (all three samples valid). + * + * Usage: + * bun run scripts/judge-calibration.ts manifest [--budget 90] + * bun run scripts/judge-calibration.ts run --config --budget [--concurrency 6] + * bun run scripts/judge-calibration.ts analyze --config + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import Anthropic from '@anthropic-ai/sdk'; +import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from '../test/helpers/llm-judge'; +import { JUDGE_MS } from '../test/helpers/eval-budgets'; +import { estimateCostUsd } from '../test/helpers/pricing'; + +export const PANEL_SAMPLES = 3; +export const PHASES = ['old-a', 'old-b', 'new'] as const; +export type Phase = typeof PHASES[number]; +export type Verdict = 'pass' | 'fail'; +export type ErrorKind = 'refusal' | 'truncation' | 'parse' | 'schema' | 'transport'; +export const LATENCY_CEILING_MS = 0.8 * JUDGE_MS; +export const CORPUS_ROOT = path.join(import.meta.dir, '..', 'test', 'fixtures', 'judge-calibration'); + +export interface CorpusItem { + id: string; + category: 'pass' | 'fail' | 'adjacent'; + expected: Verdict; + split: 'dev' | 'heldout'; + rationale: string; + inputs: Record; +} + +export interface CalibrationConfig { + id: string; + /** Dispatch order when the priced manifest exceeds the budget (armJudge, inline judges, workflow by case count). */ + priority: number; + /** Eval cases that send this request configuration. */ + cases: string[]; + model: string; + build(inputs: Record): string; + /** Today's request options. */ + oldOptions: CallJudgeOptions; + /** Comparison (1): today's options plus the JSON schema; prose unchanged. */ + newOptions: CallJudgeOptions; + /** Throws when production would reject the sample's shape. */ + validate(value: unknown): void; + verdict(samples: unknown[]): Verdict; + /** Manifest pricing estimate per call (output includes adaptive thinking). */ + estimatedOutputTokens: number; +} + +export interface SampleRecord { + config: string; + item: string; + split: CorpusItem['split']; + phase: Phase; + sample: number; + status: 'ok' | 'error'; + error_kind?: ErrorKind; + error?: string; + value?: unknown; + latency_ms: number; + stop_reason: string | null; + usage: { input_tokens: number; output_tokens: number } | null; + cost_usd: number; +} + +export type JudgeCall = (prompt: string, model: string, opts: CallJudgeOptions) => Promise; + +export function loadCorpus(configId: string, root = CORPUS_ROOT): CorpusItem[] { + const dir = path.join(root, configId); + const corpus = JSON.parse(fs.readFileSync(path.join(dir, 'corpus.json'), 'utf8')); + return corpus.items.map((item: any) => ({ + ...item, + inputs: Object.fromEntries(Object.entries(item.inputs as Record).map(([key, value]) => + [key, typeof value === 'string' ? value : fs.readFileSync(path.join(dir, value.file), 'utf8')])), + })); +} + +export function classifyError(error: unknown): ErrorKind { + if (error instanceof JudgeRefusalError) return 'refusal'; + const message = error instanceof Error ? error.message : String(error); + if (/truncated at max_tokens|stop_reason=max_tokens/.test(message)) return 'truncation'; + if (error instanceof SyntaxError || /non-JSON|did not complete/.test(message)) return 'parse'; + return 'transport'; +} + +/** The fixed request identity: what must not move between dispatch and analysis. */ +export function requestHash(config: CalibrationConfig, phase: Phase): string { + const { onResponse: _ignored, signal: _signal, ...options } = phase === 'new' ? config.newOptions : config.oldOptions; + return createHash('sha256').update(JSON.stringify({ model: config.model, options })).digest('hex'); +} + +export interface PlannedCall { item: CorpusItem; phase: Phase; sample: number } + +/** Every paid call for one configuration, interleaved by a fixed seed so phases share time-of-day drift. */ +export function planCalls(items: CorpusItem[], split: CorpusItem['split']): PlannedCall[] { + const calls = items.filter(item => item.split === split) + .flatMap(item => PHASES.flatMap(phase => Array.from({ length: PANEL_SAMPLES }, (_, sample) => ({ item, phase, sample })))); + let seed = 0x5eed; + const random = () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31; + for (let i = calls.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + [calls[i], calls[j]] = [calls[j]!, calls[i]!]; + } + return calls; +} + +export async function runSample(config: CalibrationConfig, planned: PlannedCall, call: JudgeCall = callJudge as JudgeCall, + clock: () => number = () => performance.now()): Promise { + let meta: JudgeResponseMeta | null = null; + const options = { ...(planned.phase === 'new' ? config.newOptions : config.oldOptions), onResponse: (m: JudgeResponseMeta) => { meta = m; } }; + const base = { config: config.id, item: planned.item.id, split: planned.item.split, phase: planned.phase, sample: planned.sample }; + const started = clock(); + let value: unknown; + let failure: { kind: ErrorKind; message: string } | null = null; + try { + value = await call(config.build(planned.item.inputs), config.model, options); + try { config.validate(value); } + catch (error) { failure = { kind: 'schema', message: error instanceof Error ? error.message : String(error) }; } + } catch (error) { + failure = { kind: classifyError(error), message: error instanceof Error ? error.message : String(error) }; + } + const latency_ms = Math.round(clock() - started); + const response = meta as JudgeResponseMeta | null; + const usage = response?.usage ?? null; + const cost_usd = usage ? estimateCostUsd({ input: usage.input_tokens, output: usage.output_tokens }, config.model) : 0; + const shared = { latency_ms, stop_reason: response?.stop_reason ?? null, usage, cost_usd }; + return failure + ? { ...base, status: 'error', error_kind: failure.kind, error: failure.message.slice(0, 500), ...shared } + : { ...base, status: 'ok', value, ...shared }; +} + +export interface RunOptions { + budgetUsd: number; + concurrency?: number; + call?: JudgeCall; + clock?: () => number; + onRecord?: (record: SampleRecord) => void; +} + +/** + * Dispatch planned calls with bounded concurrency. Stops dispatching when the + * actual spend plus one in-flight estimate per open slot would pass the budget, + * or after two consecutive transport failures (infrastructure, not a verdict). + */ +export async function runCalls(config: CalibrationConfig, calls: PlannedCall[], opts: RunOptions): Promise<{ records: SampleRecord[]; stopped: string | null }> { + const records: SampleRecord[] = []; + const perCall = estimateCostUsd({ input: 0, output: config.estimatedOutputTokens }, config.model); + let spent = 0, next = 0, consecutiveTransport = 0, inFlight = 0; + let stopped: string | null = null; + const worker = async () => { + while (!stopped && next < calls.length) { + if (spent + (inFlight + 1) * perCall * 2 > opts.budgetUsd) { stopped = `budget: spent $${spent.toFixed(2)} of $${opts.budgetUsd.toFixed(2)}`; break; } + const planned = calls[next++]!; + inFlight++; + const record = await runSample(config, planned, opts.call, opts.clock); + inFlight--; + spent += record.cost_usd; + consecutiveTransport = record.error_kind === 'transport' ? consecutiveTransport + 1 : 0; + records.push(record); + opts.onRecord?.(record); + if (consecutiveTransport >= 2) stopped = `infrastructure: two consecutive transport failures (${record.error})`; + } + }; + await Promise.all(Array.from({ length: Math.max(1, opts.concurrency ?? 6) }, worker)); + return { records, stopped }; +} + +export interface PanelResult { verdict: Verdict | null; errors: number } + +export function panels(config: CalibrationConfig, records: SampleRecord[]): Map> { + const byItem = new Map>(); + for (const item of new Set(records.map(record => record.item))) { + const entry = {} as Record; + for (const phase of PHASES) { + const samples = records.filter(record => record.item === item && record.phase === phase); + const ok = samples.filter(record => record.status === 'ok'); + entry[phase] = { verdict: ok.length === PANEL_SAMPLES && samples.length === PANEL_SAMPLES ? config.verdict(ok.map(record => record.value)) : null, + errors: samples.length - ok.length }; + } + byItem.set(item, entry); + } + return byItem; +} + +function p95(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(0.95 * sorted.length) - 1)]!; +} + +export interface ComparisonSummary { + items: number; + old_flips: { flips: number; compared: number }; + new_flips: { flips: number; compared: number }; + agreement: Record; + false_passes: Record; + new_false_passes: string[]; +} + +export function compare(config: CalibrationConfig, items: CorpusItem[], records: SampleRecord[]): ComparisonSummary { + const byItem = panels(config, records); + const flips = (a: Phase, b: Phase) => { + const both = items.map(item => byItem.get(item.id)).filter(entry => entry?.[a].verdict && entry[b].verdict); + return { flips: both.filter(entry => entry![a].verdict !== entry![b].verdict).length, compared: both.length }; + }; + const agreement = Object.fromEntries(PHASES.map(phase => + [phase, items.filter(item => byItem.get(item.id)?.[phase].verdict === item.expected).length])) as Record; + const falsePasses = Object.fromEntries(PHASES.map(phase => + [phase, items.filter(item => item.expected === 'fail' && byItem.get(item.id)?.[phase].verdict === 'pass').map(item => item.id)])) as Record; + return { items: items.length, old_flips: flips('old-a', 'old-b'), new_flips: flips('new', 'old-a'), agreement, + false_passes: falsePasses, new_false_passes: falsePasses.new.filter(id => !falsePasses['old-a'].includes(id)) }; +} + +export interface CalibrationResult { + config: string; + model: string; + request_hash: Record<'old' | 'new', string>; + complete: boolean; + stopped: string | null; + calls: number; + cost_usd: number; + errors: Record<'old' | 'new', Record & { samples: number; rate: number }>; + latency_p95_ms: Record<'old' | 'new', number | null>; + all: ComparisonSummary; + heldout: ComparisonSummary; + landing: { lands: boolean; reasons: string[]; strict_no_flip_increase: boolean }; + prose_step_triggered: { triggered: boolean; reasons: string[] }; +} + +export function analyze(config: CalibrationConfig, items: CorpusItem[], records: SampleRecord[], stopped: string | null = null): CalibrationResult { + const prompt = (phase: Phase) => phase === 'new' ? 'new' : 'old'; + const errors = Object.fromEntries((['old', 'new'] as const).map(which => { + const samples = records.filter(record => prompt(record.phase) === which); + const counts = Object.fromEntries((['refusal', 'truncation', 'parse', 'schema', 'transport'] as const) + .map(kind => [kind, samples.filter(record => record.error_kind === kind).length])) as Record; + const errored = samples.filter(record => record.status === 'error').length; + return [which, { ...counts, samples: samples.length, rate: samples.length ? errored / samples.length : 0 }]; + })) as CalibrationResult['errors']; + const latency = Object.fromEntries((['old', 'new'] as const).map(which => + [which, p95(records.filter(record => prompt(record.phase) === which && record.status === 'ok').map(record => record.latency_ms))])) as CalibrationResult['latency_p95_ms']; + const expectedCalls = items.length * PHASES.length * PANEL_SAMPLES; + const complete = records.length === expectedCalls && !stopped; + const all = compare(config, items, records); + const heldout = compare(config, items.filter(item => item.split === 'heldout'), records); + + const reasons: string[] = []; + if (!complete) reasons.push(`incomplete: ${records.length}/${expectedCalls} calls${stopped ? ` (${stopped})` : ''}`); + if (all.new_flips.flips > all.old_flips.flips + 1) reasons.push(`new flips ${all.new_flips.flips}/${all.new_flips.compared} > old flips ${all.old_flips.flips}/${all.old_flips.compared} + 1`); + if (all.agreement.new < all.agreement['old-a']) reasons.push(`expected-verdict agreement dropped: new ${all.agreement.new} < old ${all.agreement['old-a']}`); + if (all.new_false_passes.length > 0) reasons.push(`new false passes: ${all.new_false_passes.join(', ')}`); + if (latency.new !== null && latency.new > LATENCY_CEILING_MS) reasons.push(`new p95 latency ${latency.new} ms > ${LATENCY_CEILING_MS} ms (80% of JUDGE_MS)`); + + const trigger: string[] = []; + if (records.some(record => record.phase === 'new' && record.status === 'error')) trigger.push('new prompt has errored samples'); + if (reasons.length > 0) trigger.push('a landing condition failed'); + if (all.old_flips.flips >= 3) trigger.push(`old-vs-old flips ${all.old_flips.flips} >= 3`); + + return { + config: config.id, model: config.model, + request_hash: { old: requestHash(config, 'old-a'), new: requestHash(config, 'new') }, + complete, stopped, calls: records.length, + cost_usd: +records.reduce((sum, record) => sum + record.cost_usd, 0).toFixed(4), + errors, latency_p95_ms: latency, all, heldout, + landing: { lands: reasons.length === 0, reasons, strict_no_flip_increase: all.new_flips.flips <= all.old_flips.flips }, + prose_step_triggered: { triggered: trigger.length > 0, reasons: trigger }, + }; +} + +export interface ManifestEntry { + config: string; + priority: number; + cases: number; + model: string; + items: number; + calls: number; + input_tokens: number; + estimated_output_tokens_per_call: number; + estimated_cost_usd: number; +} + +/** Price every planned call before dispatch; count_tokens is free. */ +export async function priceManifest(configs: CalibrationConfig[], countTokens: (model: string, prompt: string) => Promise, + root = CORPUS_ROOT): Promise { + const entries: ManifestEntry[] = []; + for (const config of [...configs].sort((a, b) => a.priority - b.priority)) { + if (!fs.existsSync(path.join(root, config.id, 'corpus.json'))) continue; + const items = loadCorpus(config.id, root); + let input = 0; + for (const item of items) input += await countTokens(config.model, config.build(item.inputs)) * PHASES.length * PANEL_SAMPLES; + const calls = items.length * PHASES.length * PANEL_SAMPLES; + entries.push({ config: config.id, priority: config.priority, cases: config.cases.length, model: config.model, items: items.length, calls, + input_tokens: input, estimated_output_tokens_per_call: config.estimatedOutputTokens, + estimated_cost_usd: estimateCostUsd({ input, output: calls * config.estimatedOutputTokens }, config.model) }); + } + return entries; +} + +function flag(args: string[], name: string): string | undefined { + const index = args.indexOf(`--${name}`); + return index >= 0 ? args[index + 1] : undefined; +} + +async function main(args: string[]): Promise { + // The per-judge table (builders, schemas, verdict rules) is committed separately from this harness. + const { CALIBRATION_CONFIGS } = await import(path.join(import.meta.dir, 'judge-calibration-configs.ts')) as { CALIBRATION_CONFIGS: CalibrationConfig[] }; + const command = args[0]; + if (command === 'manifest') { + const client = new Anthropic(); + const entries = await priceManifest(CALIBRATION_CONFIGS, async (model, prompt) => + (await client.messages.countTokens({ model, messages: [{ role: 'user', content: prompt }] })).input_tokens); + const budget = Number(flag(args, 'budget') ?? 90); + let running = 0; + const manifest = { generated_at: new Date().toISOString(), budget_usd: budget, + entries: entries.map(entry => ({ ...entry, cumulative_usd: +(running += entry.estimated_cost_usd).toFixed(4), within_budget: running <= budget })) }; + fs.writeFileSync(path.join(CORPUS_ROOT, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n'); + console.table(manifest.entries); + return; + } + const config = CALIBRATION_CONFIGS.find(candidate => candidate.id === flag(args, 'config')); + if (!config) throw new Error(`unknown --config; expected one of ${CALIBRATION_CONFIGS.map(candidate => candidate.id).join(', ')}`); + const items = loadCorpus(config.id); + const runsDir = path.join(CORPUS_ROOT, config.id, 'runs'); + const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl'); + const resultPath = path.join(runsDir, 'comparison-1.result.json'); + if (command === 'run') { + if (fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`); + fs.mkdirSync(runsDir, { recursive: true }); + const budgetUsd = Number(flag(args, 'budget')); + if (!Number.isFinite(budgetUsd) || budgetUsd <= 0) throw new Error('--budget is required'); + const concurrency = Number(flag(args, 'concurrency') ?? 6); + const onRecord = (record: SampleRecord) => fs.appendFileSync(samplesPath, JSON.stringify(record) + '\n'); + let spent = 0, stopped: string | null = null; + // Development items first; the held-out third is dispatched only after, with the same frozen requests. + for (const split of ['dev', 'heldout'] as const) { + const outcome = await runCalls(config, planCalls(items, split), { budgetUsd: budgetUsd - spent, concurrency, onRecord }); + spent += outcome.records.reduce((sum, record) => sum + record.cost_usd, 0); + console.log(`[calibration] ${config.id} ${split}: ${outcome.records.length} calls, $${spent.toFixed(2)} cumulative${outcome.stopped ? `, stopped: ${outcome.stopped}` : ''}`); + if ((stopped = outcome.stopped)) break; + } + fs.writeFileSync(path.join(runsDir, 'comparison-1.stop.json'), JSON.stringify({ stopped }) + '\n'); + } else if (command !== 'analyze') { + throw new Error(`unknown command ${command}; expected manifest, run or analyze`); + } + const records: SampleRecord[] = fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + const stopPath = path.join(runsDir, 'comparison-1.stop.json'); + const stopped = fs.existsSync(stopPath) ? JSON.parse(fs.readFileSync(stopPath, 'utf8')).stopped : null; + const result = analyze(config, items, records, stopped); + fs.writeFileSync(resultPath, JSON.stringify(result, null, 2) + '\n'); + console.log(JSON.stringify(result, null, 2)); +} + +if (import.meta.main) { + main(process.argv.slice(2)).catch(error => { console.error(error); process.exit(1); }); +} diff --git a/test/helpers/llm-judge.ts b/test/helpers/llm-judge.ts index f06016429..498854517 100644 --- a/test/helpers/llm-judge.ts +++ b/test/helpers/llm-judge.ts @@ -106,6 +106,15 @@ export interface CallJudgeOptions { jsonSchema?: JSONOutputFormat['schema']; /** Adaptive-thinking effort; the judge models accept no thinking token budget. */ effort?: 'low' | 'medium' | 'high'; + /** Observes the provider response before parsing (calibration cost and stop accounting); never sent. */ + onResponse?: (response: JudgeResponseMeta) => void; +} + +export interface JudgeResponseMeta { + id: string | null; + model: string | null; + stop_reason: string | null; + usage: { input_tokens: number; output_tokens: number } | null; } export async function callJudge( @@ -166,6 +175,13 @@ export async function callJudge( } } + opts?.onResponse?.({ + id: typeof response.id === 'string' ? response.id : null, + model: typeof response.model === 'string' ? response.model : null, + stop_reason: response.stop_reason ?? null, + usage: typeof response.usage?.input_tokens === 'number' && typeof response.usage?.output_tokens === 'number' + ? { input_tokens: response.usage.input_tokens, output_tokens: response.usage.output_tokens } : null, + }); const text = response.content .filter(block => block.type === 'text') .map(block => block.text) diff --git a/test/judge-calibration.test.ts b/test/judge-calibration.test.ts new file mode 100644 index 000000000..f2f2c5139 --- /dev/null +++ b/test/judge-calibration.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, spyOn, test } from 'bun:test'; +import Anthropic from '@anthropic-ai/sdk'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge'; +import { + analyze, classifyError, loadCorpus, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample, + type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord, +} from '../scripts/judge-calibration'; + +const item = (id: string, expected: 'pass' | 'fail', split: 'dev' | 'heldout' = 'dev'): CorpusItem => + ({ id, category: expected, expected, split, rationale: 'fixture', inputs: { text: `doc ${id}` } }); + +const config: CalibrationConfig = { + id: 'stub', priority: 1, cases: ['a', 'b'], model: 'claude-fable-5-1', + build: inputs => `grade: ${inputs.text}`, + oldOptions: {}, + newOptions: { jsonSchema: { type: 'object' } }, + validate: value => { if (typeof (value as { score?: unknown })?.score !== 'number') throw new Error('score missing'); }, + verdict: samples => samples.filter(sample => (sample as { score: number }).score >= 4).length * 2 > samples.length ? 'pass' : 'fail', + estimatedOutputTokens: 1000, +}; + +/** Stub provider keyed by prompt and phase; usage is reported through the real onResponse seam. */ +function stub(score: (prompt: string, opts: CallJudgeOptions, n: number) => unknown): { call: JudgeCall; calls: Array<{ prompt: string; opts: CallJudgeOptions }> } { + const calls: Array<{ prompt: string; opts: CallJudgeOptions }> = []; + return { + calls, + call: async (prompt, _model, opts) => { + calls.push({ prompt, opts }); + opts.onResponse?.({ id: 'msg', model: 'claude-fable-5-1', stop_reason: 'end_turn', usage: { input_tokens: 1000, output_tokens: 1000 } }); + const value = score(prompt, opts, calls.length); + if (value instanceof Error) throw value; + return value; + }, + }; +} + +const records = async (items: CorpusItem[], call: JudgeCall) => { + const out: SampleRecord[] = []; + for (const split of ['dev', 'heldout'] as const) out.push(...(await runCalls(config, planCalls(items, split), { budgetUsd: 1000, call })).records); + return out; +}; + +describe('judge calibration harness (free, stubbed provider)', () => { + test('every item gets two independent old panels and one new panel, each sample a direct call', async () => { + const items = [item('p1', 'pass'), item('f1', 'fail'), item('h1', 'pass', 'heldout')]; + const provider = stub(() => ({ score: 5 })); + const out = await records(items, provider.call); + expect(provider.calls).toHaveLength(items.length * PHASES.length * PANEL_SAMPLES); + for (const entry of items) { + for (const phase of PHASES) expect(out.filter(record => record.item === entry.id && record.phase === phase)).toHaveLength(PANEL_SAMPLES); + } + expect(provider.calls.filter(call => call.opts.jsonSchema)).toHaveLength(items.length * PANEL_SAMPLES); + expect(new Set(provider.calls.map(call => call.prompt))).toEqual(new Set(items.map(entry => `grade: ${entry.inputs.text}`))); + // Held-out samples are dispatched only after every development sample. + const firstHeldout = out.findIndex(record => record.split === 'heldout'); + expect(out.slice(firstHeldout).every(record => record.split === 'heldout')).toBe(true); + }); + + test('the harness never routes through judgePanel or the workflow passing-panel cache', () => { + const scripts = path.join(import.meta.dir, '..', 'scripts'); + for (const file of ['judge-calibration.ts', 'judge-calibration-configs.ts'].filter(name => fs.existsSync(path.join(scripts, name)))) { + const source = fs.readFileSync(path.join(scripts, file), 'utf8'); + const imports = source.match(/^import[\s\S]*?from '[^']+';$/gm) ?? []; + expect(imports.length).toBeGreaterThan(0); + expect(imports.join('\n')).not.toMatch(/\bjudgePanel\b|workflow-judge-cache|eval-input-cache/); + expect(source.replace(/^\s*(?:\/\*\*|\*|\/\/).*$/gm, '')).not.toMatch(/\bjudgePanel\(|prepareWorkflowJudgeCache|lookupEvalInputCache/); + } + }); + + test('a second old panel is sampled, not replayed: old-vs-old flips are counted', async () => { + const items = [item('p1', 'pass')]; + const provider = stub((_prompt, opts, n) => ({ score: opts.jsonSchema ? 5 : n <= 3 ? 5 : 2 })); + // Force the old panels apart: the first three old calls pass, the next three fail. + let oldCalls = 0; + const call: JudgeCall = (prompt, model, opts) => provider.call(prompt, model, opts).then(value => + opts.jsonSchema ? value : ({ score: ++oldCalls <= 3 ? 5 : 2 })); + const result = analyze(config, items, await records(items, call)); + expect(result.all.old_flips).toEqual({ flips: 1, compared: 1 }); + }); + + test('error kinds are recorded per sample and never count as verdicts', async () => { + expect(classifyError(new JudgeRefusalError({ content: [] }))).toBe('refusal'); + expect(classifyError(new Error('Judge response truncated at max_tokens=8192 (model=m)'))).toBe('truncation'); + expect(classifyError(new SyntaxError('Unexpected token'))).toBe('parse'); + expect(classifyError(new Error('Judge returned non-JSON: hi'))).toBe('parse'); + expect(classifyError(new Error('Structured judge did not complete: stop_reason=pause_turn'))).toBe('parse'); + expect(classifyError(Object.assign(new Error('Overloaded'), { status: 529 }))).toBe('transport'); + + const planned = { item: item('p1', 'pass'), phase: 'new' as const, sample: 0 }; + const schema = await runSample(config, planned, stub(() => ({ verdict: 'yes' })).call); + expect(schema).toMatchObject({ status: 'error', error_kind: 'schema', cost_usd: 0.06 }); + expect(schema).not.toHaveProperty('value'); + const refusal = await runSample(config, planned, stub(() => new JudgeRefusalError({ content: [] })).call); + expect(refusal).toMatchObject({ status: 'error', error_kind: 'refusal' }); + + // One errored new sample leaves that panel incomplete: excluded from flips, counted as an error. + const items = [item('p1', 'pass'), item('p2', 'pass')]; + let newCalls = 0; + const result = analyze(config, items, await records(items, stub((_prompt, opts) => + opts.jsonSchema && ++newCalls === 1 ? new SyntaxError('bad json') : { score: 5 }).call)); + expect(result.errors.new).toMatchObject({ parse: 1, samples: 6 }); + expect(result.errors.old).toMatchObject({ parse: 0, samples: 12, rate: 0 }); + expect(result.all.new_flips.compared).toBe(1); + expect(result.all.old_flips.compared).toBe(2); + expect(result.prose_step_triggered.triggered).toBe(true); + expect(result.prose_step_triggered.reasons).toContain('new prompt has errored samples'); + }); + + test('latency is measured per sample and the p95 gate holds the schema back', async () => { + let now = 0; + const clock = () => now; + const slow: JudgeCall = async (_prompt, _model, opts) => { now += opts.jsonSchema ? 100_000 : 10_000; return { score: 5 }; }; + const items = [item('p1', 'pass')]; + const out: SampleRecord[] = (await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 1000, call: slow, clock, concurrency: 1 })).records; + expect(out.filter(record => record.phase === 'new').every(record => record.latency_ms === 100_000)).toBe(true); + const result = analyze(config, items, out); + expect(result.latency_p95_ms).toEqual({ old: 10_000, new: 100_000 }); + expect(result.landing.lands).toBe(false); + expect(result.landing.reasons.join(' ')).toContain('p95 latency'); + }); + + test('landing: flips at most old+1, agreement does not drop, zero new false passes', async () => { + const items = [item('p1', 'pass'), item('f1', 'fail'), item('f2', 'fail')]; + const same = analyze(config, items, await records(items, stub(prompt => ({ score: prompt.includes('p1') ? 5 : 1 })).call)); + expect(same.landing).toEqual({ lands: true, reasons: [], strict_no_flip_increase: true }); + expect(same.all.agreement).toEqual({ 'old-a': 3, 'old-b': 3, new: 3 }); + + const leaky = analyze(config, items, await records(items, stub((prompt, opts) => + ({ score: prompt.includes('p1') || (opts.jsonSchema && prompt.includes('f1')) ? 5 : 1 })).call)); + expect(leaky.all.new_false_passes).toEqual(['f1']); + expect(leaky.landing.lands).toBe(false); + expect(leaky.landing.reasons.join(' ')).toContain('new false passes: f1'); + expect(leaky.landing.reasons.join(' ')).toContain('agreement dropped'); + }); + + test('two consecutive transport failures stop the study; the budget stops dispatch before it is passed', async () => { + const items = [item('p1', 'pass'), item('p2', 'pass')]; + const down = await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 1000, concurrency: 1, + call: stub(() => Object.assign(new Error('socket hang up'), { status: 500 })).call }); + expect(down.records).toHaveLength(2); + expect(down.stopped).toContain('infrastructure'); + + const capped = await runCalls(config, planCalls(items, 'dev'), { budgetUsd: 0.5, concurrency: 1, call: stub(() => ({ score: 5 })).call }); + expect(capped.stopped).toContain('budget'); + expect(capped.records.reduce((sum, record) => sum + record.cost_usd, 0)).toBeLessThanOrEqual(0.5); + const result = analyze(config, items, capped.records, capped.stopped); + expect(result.complete).toBe(false); + expect(result.landing.lands).toBe(false); + }); + + test('callJudge reports usage and stop reason through onResponse without sending it', async () => { + const key = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = 'test-only-key'; + const create = spyOn(Anthropic.Messages.prototype, 'create').mockResolvedValue({ id: 'msg_1', model: 'claude-fable-5-1', + stop_reason: 'end_turn', content: [{ type: 'text', text: '{"score":4}' }], usage: { input_tokens: 12, output_tokens: 34 } } as never); + try { + const seen: JudgeResponseMeta[] = []; + expect(await callJudge<{ score: number }>('grade', 'claude-fable-5-1', { onResponse: meta => seen.push(meta) })).toEqual({ score: 4 }); + expect(seen).toEqual([{ id: 'msg_1', model: 'claude-fable-5-1', stop_reason: 'end_turn', usage: { input_tokens: 12, output_tokens: 34 } }]); + expect(create.mock.calls[0]![0]).toEqual({ model: 'claude-fable-5-1', max_tokens: 8192, messages: [{ role: 'user', content: 'grade' }] }); + } finally { + create.mockRestore(); + if (key === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = key; + } + }); + + test('the manifest prices every call in priority order from a committed corpus', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'judge-cal-')); + try { + for (const id of ['second', 'first']) { + fs.mkdirSync(path.join(root, id, 'inputs'), { recursive: true }); + fs.writeFileSync(path.join(root, id, 'inputs', 'p1.text.md'), 'from file'); + fs.writeFileSync(path.join(root, id, 'corpus.json'), JSON.stringify({ items: [ + { ...item('p1', 'pass'), inputs: { text: { file: 'inputs/p1.text.md' } } }, item('f1', 'fail', 'heldout')] })); + } + expect(loadCorpus('first', root)[0]!.inputs.text).toBe('from file'); + const entries = await priceManifest([{ ...config, id: 'second', priority: 2 }, { ...config, id: 'first', priority: 1 }, { ...config, id: 'absent', priority: 0 }], + async () => 2000, root); + expect(entries.map(entry => entry.config)).toEqual(['first', 'second']); + expect(entries[0]).toMatchObject({ items: 2, calls: 18, input_tokens: 36_000, estimated_cost_usd: 1.26 }); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }); +});