diff --git a/scripts/judge-calibration.ts b/scripts/judge-calibration.ts index 68e8fae95..9b376c10a 100644 --- a/scripts/judge-calibration.ts +++ b/scripts/judge-calibration.ts @@ -16,6 +16,7 @@ * Usage: * bun run scripts/judge-calibration.ts manifest [--budget 90] * bun run scripts/judge-calibration.ts run --config --budget [--concurrency 6] + * bun run scripts/judge-calibration.ts resume --config --budget (only never-dispatched planned calls) * bun run scripts/judge-calibration.ts analyze --config */ import * as fs from 'node:fs'; @@ -119,6 +120,12 @@ export function planCalls(items: CorpusItem[], split: CorpusItem['split']): Plan return calls; } +/** Planned calls never dispatched (a budget or infrastructure stop); completing them is not a resample. */ +export function undispatched(calls: PlannedCall[], records: SampleRecord[]): PlannedCall[] { + const done = new Set(records.map(record => `${record.item}/${record.phase}/${record.sample}`)); + return calls.filter(call => !done.has(`${call.item.id}/${call.phase}/${call.sample}`)); +} + export async function runSample(config: CalibrationConfig, planned: PlannedCall, call: JudgeCall = callJudge as JudgeCall, clock: () => number = () => performance.now()): Promise { let meta: JudgeResponseMeta | null = null; @@ -337,8 +344,11 @@ async function main(args: string[]): Promise { const runsDir = path.join(CORPUS_ROOT, config.id, 'runs'); const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl'); const resultPath = path.join(runsDir, 'comparison-1.result.json'); - if (command === 'run') { - if (fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`); + if (command === 'run' || command === 'resume') { + // resume dispatches only planned calls a stop left undispatched; recorded samples are never redrawn. + if (command === 'run' && fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`); + if (command === 'resume' && !fs.existsSync(samplesPath)) throw new Error(`${samplesPath} is missing; nothing to resume`); + const prior: SampleRecord[] = command === 'resume' ? fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)) : []; fs.mkdirSync(runsDir, { recursive: true }); const budgetUsd = Number(flag(args, 'budget')); if (!Number.isFinite(budgetUsd) || budgetUsd <= 0) throw new Error('--budget is required'); @@ -347,14 +357,14 @@ async function main(args: string[]): Promise { let spent = 0, stopped: string | null = null; // Development items first; the held-out third is dispatched only after, with the same frozen requests. for (const split of ['dev', 'heldout'] as const) { - const outcome = await runCalls(config, planCalls(items, split), { budgetUsd: budgetUsd - spent, concurrency, onRecord }); + const outcome = await runCalls(config, undispatched(planCalls(items, split), prior), { budgetUsd: budgetUsd - spent, concurrency, onRecord }); spent += outcome.records.reduce((sum, record) => sum + record.cost_usd, 0); console.log(`[calibration] ${config.id} ${split}: ${outcome.records.length} calls, $${spent.toFixed(2)} cumulative${outcome.stopped ? `, stopped: ${outcome.stopped}` : ''}`); if ((stopped = outcome.stopped)) break; } fs.writeFileSync(path.join(runsDir, 'comparison-1.stop.json'), JSON.stringify({ stopped }) + '\n'); } else if (command !== 'analyze') { - throw new Error(`unknown command ${command}; expected manifest, run or analyze`); + throw new Error(`unknown command ${command}; expected manifest, run, resume or analyze`); } const records: SampleRecord[] = fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)); const stopPath = path.join(runsDir, 'comparison-1.stop.json'); diff --git a/test/judge-calibration.test.ts b/test/judge-calibration.test.ts index 942812b81..b4d5671de 100644 --- a/test/judge-calibration.test.ts +++ b/test/judge-calibration.test.ts @@ -5,7 +5,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge'; import { - analyze, classifyError, loadCorpus, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample, + analyze, classifyError, loadCorpus, undispatched, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample, type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord, } from '../scripts/judge-calibration'; @@ -151,6 +151,19 @@ describe('judge calibration harness (free, stubbed provider)', () => { expect(result.landing.lands).toBe(false); }); + test('resume completes only never-dispatched planned calls and never redraws a recorded sample', async () => { + const items = [item('p1', 'pass'), item('p2', 'pass')]; + const planned = planCalls(items, 'dev'); + const first = await runCalls(config, planned.slice(0, 7), { budgetUsd: 1000, call: stub(() => ({ score: 5 })).call }); + const rest = undispatched(planned, first.records); + expect(rest).toEqual(planned.slice(7)); + const provider = stub(() => ({ score: 5 })); + const second = await runCalls(config, rest, { budgetUsd: 1000, call: provider.call }); + expect(provider.calls).toHaveLength(planned.length - 7); + expect(undispatched(planned, [...first.records, ...second.records])).toEqual([]); + expect(analyze(config, items, [...first.records, ...second.records]).complete).toBe(true); + }); + test('callJudge reports usage and stop reason through onResponse without sending it', async () => { const key = process.env.ANTHROPIC_API_KEY; process.env.ANTHROPIC_API_KEY = 'test-only-key';