test: calibration resume dispatches only never-dispatched planned calls (W2 R6)

The cross-skill run stopped at its per-config budget backstop with 24
held-out calls undispatched. resume completes exactly the planned calls
that have no record; recorded samples are never redrawn.

scripts/judge-calibration.ts

@@ -16,6 +16,7 @@
  * Usage:
  *   bun run scripts/judge-calibration.ts manifest [--budget 90]
  *   bun run scripts/judge-calibration.ts run --config <id> --budget <usd> [--concurrency 6]
+ *   bun run scripts/judge-calibration.ts resume --config <id> --budget <usd>   (only never-dispatched planned calls)
  *   bun run scripts/judge-calibration.ts analyze --config <id>
  */
 import * as fs from 'node:fs';
@@ -119,6 +120,12 @@ export function planCalls(items: CorpusItem[], split: CorpusItem['split']): Plan
   return calls;
 }
 
+/** Planned calls never dispatched (a budget or infrastructure stop); completing them is not a resample. */
+export function undispatched(calls: PlannedCall[], records: SampleRecord[]): PlannedCall[] {
+  const done = new Set(records.map(record => `${record.item}/${record.phase}/${record.sample}`));
+  return calls.filter(call => !done.has(`${call.item.id}/${call.phase}/${call.sample}`));
+}
+
 export async function runSample(config: CalibrationConfig, planned: PlannedCall, call: JudgeCall = callJudge as JudgeCall,
   clock: () => number = () => performance.now()): Promise<SampleRecord> {
   let meta: JudgeResponseMeta | null = null;
@@ -337,8 +344,11 @@ async function main(args: string[]): Promise<void> {
   const runsDir = path.join(CORPUS_ROOT, config.id, 'runs');
   const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl');
   const resultPath = path.join(runsDir, 'comparison-1.result.json');
-  if (command === 'run') {
-    if (fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`);
+  if (command === 'run' || command === 'resume') {
+    // resume dispatches only planned calls a stop left undispatched; recorded samples are never redrawn.
+    if (command === 'run' && fs.existsSync(samplesPath)) throw new Error(`${samplesPath} exists; calibration runs are never repeated`);
+    if (command === 'resume' && !fs.existsSync(samplesPath)) throw new Error(`${samplesPath} is missing; nothing to resume`);
+    const prior: SampleRecord[] = command === 'resume' ? fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line)) : [];
     fs.mkdirSync(runsDir, { recursive: true });
     const budgetUsd = Number(flag(args, 'budget'));
     if (!Number.isFinite(budgetUsd) || budgetUsd <= 0) throw new Error('--budget <usd> is required');
@@ -347,14 +357,14 @@ async function main(args: string[]): Promise<void> {
     let spent = 0, stopped: string | null = null;
     // Development items first; the held-out third is dispatched only after, with the same frozen requests.
     for (const split of ['dev', 'heldout'] as const) {
-      const outcome = await runCalls(config, planCalls(items, split), { budgetUsd: budgetUsd - spent, concurrency, onRecord });
+      const outcome = await runCalls(config, undispatched(planCalls(items, split), prior), { budgetUsd: budgetUsd - spent, concurrency, onRecord });
       spent += outcome.records.reduce((sum, record) => sum + record.cost_usd, 0);
       console.log(`[calibration] ${config.id} ${split}: ${outcome.records.length} calls, $${spent.toFixed(2)} cumulative${outcome.stopped ? `, stopped: ${outcome.stopped}` : ''}`);
       if ((stopped = outcome.stopped)) break;
     }
     fs.writeFileSync(path.join(runsDir, 'comparison-1.stop.json'), JSON.stringify({ stopped }) + '\n');
   } else if (command !== 'analyze') {
-    throw new Error(`unknown command ${command}; expected manifest, run or analyze`);
+    throw new Error(`unknown command ${command}; expected manifest, run, resume or analyze`);
   }
   const records: SampleRecord[] = fs.readFileSync(samplesPath, 'utf8').trim().split('\n').map(line => JSON.parse(line));
   const stopPath = path.join(runsDir, 'comparison-1.stop.json');

test/judge-calibration.test.ts

@@ -5,7 +5,7 @@ import * as os from 'node:os';
 import * as path from 'node:path';
 import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge';
 import {
-  analyze, classifyError, loadCorpus, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
+  analyze, classifyError, loadCorpus, undispatched, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
   type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord,
 } from '../scripts/judge-calibration';
 
@@ -151,6 +151,19 @@ describe('judge calibration harness (free, stubbed provider)', () => {
     expect(result.landing.lands).toBe(false);
   });
 
+  test('resume completes only never-dispatched planned calls and never redraws a recorded sample', async () => {
+    const items = [item('p1', 'pass'), item('p2', 'pass')];
+    const planned = planCalls(items, 'dev');
+    const first = await runCalls(config, planned.slice(0, 7), { budgetUsd: 1000, call: stub(() => ({ score: 5 })).call });
+    const rest = undispatched(planned, first.records);
+    expect(rest).toEqual(planned.slice(7));
+    const provider = stub(() => ({ score: 5 }));
+    const second = await runCalls(config, rest, { budgetUsd: 1000, call: provider.call });
+    expect(provider.calls).toHaveLength(planned.length - 7);
+    expect(undispatched(planned, [...first.records, ...second.records])).toEqual([]);
+    expect(analyze(config, items, [...first.records, ...second.records]).complete).toBe(true);
+  });
+
   test('callJudge reports usage and stop reason through onResponse without sending it', async () => {
     const key = process.env.ANTHROPIC_API_KEY;
     process.env.ANTHROPIC_API_KEY = 'test-only-key';