test: calibration resume dispatches only never-dispatched planned calls (W2 R6)

The cross-skill run stopped at its per-config budget backstop with 24
held-out calls undispatched. resume completes exactly the planned calls
that have no record; recorded samples are never redrawn.
This commit is contained in:
garrytan committed 2026-10-03 21:29:40 +00:00
1 parent 69cf188e39
commit b344ef6a25
2 files changed
+28 -5

No files matched your search

+14 -1
View File
@@ -5,7 +5,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge';
import {
analyze, classifyError, loadCorpus, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
analyze, classifyError, loadCorpus, undispatched, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord,
} from '../scripts/judge-calibration';
@@ -151,6 +151,19 @@ describe('judge calibration harness (free, stubbed provider)', () => {
expect(result.landing.lands).toBe(false);
});
test('resume completes only never-dispatched planned calls and never redraws a recorded sample', async () => {
const items = [item('p1', 'pass'), item('p2', 'pass')];
const planned = planCalls(items, 'dev');
const first = await runCalls(config, planned.slice(0, 7), { budgetUsd: 1000, call: stub(() => ({ score: 5 })).call });
const rest = undispatched(planned, first.records);
expect(rest).toEqual(planned.slice(7));
const provider = stub(() => ({ score: 5 }));
const second = await runCalls(config, rest, { budgetUsd: 1000, call: provider.call });
expect(provider.calls).toHaveLength(planned.length - 7);
expect(undispatched(planned, [...first.records, ...second.records])).toEqual([]);
expect(analyze(config, items, [...first.records, ...second.records]).complete).toBe(true);
});
test('callJudge reports usage and stop reason through onResponse without sending it', async () => {
const key = process.env.ANTHROPIC_API_KEY;
process.env.ANTHROPIC_API_KEY = 'test-only-key';