diff --git a/test/fixtures/judge-calibration/README.md b/test/fixtures/judge-calibration/README.md index 4a8d00184..f068bfffd 100644 --- a/test/fixtures/judge-calibration/README.md +++ b/test/fixtures/judge-calibration/README.md @@ -20,3 +20,28 @@ defines each configuration. Run outputs land in `/runs/`: and priced usage, and `comparison-1.result.json` holds the flip rates, agreement, false passes, error rates, p95 latency, landing decision and the UC3 prose-step trigger. `manifest.json` prices every planned call before dispatch. + +## Comparison (1) results (2026-10-03, `claude-fable-5-1`) + +Flips are panel-verdict differences on items where both panels completed: old-vs-old +is `old-a` vs `old-b`, new-vs-old is `new` vs `old-a`. Agreement is items whose panel +verdict matches `expected`. A configuration lands when new flips are at most old flips +plus one, agreement does not drop, there are zero new false passes and the new p95 is at +most 80% of `JUDGE_MS` (96 s). + +| Config | Old-vs-old flips | New-vs-old flips | Agreement old / new | New false passes | Errored samples old / new | p95 s old / new | Held-out new flips, agreement | Cost | Decision | +|---|---|---|---|---|---|---|---|---|---| +| `arm` | 0/24 | 0/24 | 24 / 24 | 0 | 0 / 0 | 7.6 / 5.9 | 0/8, 8/8 | $6.16 | lands | +| `qa-workflow` | 2/24 | 4/24 | 17 / 17 | 1 (a06) | 0 / 0 | 26.7 / 27.6 | 1/8, 6/8 | $18.94 | held: new flips above old + 1, one new false pass | +| `qa-health-rubric` | 0/24 | 1/24 | 22 / 23 | 0 | 0 / 0 | 17.0 / 18.7 | 0/8, 8/8 | $10.62 | lands | +| `qa-anti-refusal` | 0/24 | 0/24 | 23 / 23 | 0 | 0 / 0 | 12.5 / 13.4 | 0/8, 8/8 | $7.02 | lands | +| `cross-skill` | 0/24 | 0/24 | 23 / 23 | 0 | 0 / 0 | 19.7 / 22.6 | 0/8, 8/8 | $19.86 | lands | +| `voice` | 0/24 | 0/24 | 22 / 22 | 0 | 0 / 0 | 13.3 / 13.7 | 0/8, 8/8 | $8.09 | lands | + +No sample errored (refusal, truncation, parse, schema or transport) under either prompt. +`workflow-default` has a corpus but was not dispatched: its priced cost (about $20) exceeded +what remained of the W2 reservation after the higher-priority configurations, so the shared +workflow judge stays on today's request (only `ship` sends `WORKFLOW_JUDGE_RESPONSE_SCHEMA`, +as before). The frontier (`review`) and cookie workflow configurations have no corpus yet. +A $0.12 two-call pilot sized output tokens before the manifest was priced. Total W2 spend: +$70.81 of $90. diff --git a/test/judge-calibration.test.ts b/test/judge-calibration.test.ts index b4d5671de..6652d8762 100644 --- a/test/judge-calibration.test.ts +++ b/test/judge-calibration.test.ts @@ -253,3 +253,35 @@ describe('committed calibration configurations and corpora', () => { expect(byId.voice!.verdict([voice(4), voice(4), voice(3)])).toBe('fail'); }); }); + +describe('landed schemas match the recorded calibration', () => { + const evalSource = fs.readFileSync(path.join(import.meta.dir, 'skill-llm-eval.test.ts'), 'utf8'); + const judgeSource = fs.readFileSync(path.join(import.meta.dir, 'helpers', 'llm-judge.ts'), 'utf8'); + const result = (id: string) => JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'judge-calibration', id, 'runs', 'comparison-1.result.json'), 'utf8')); + const wiring: Array<[string, string, string]> = [ + ['qa-workflow', 'buildQaWorkflowJudgePrompt(section)', 'JUDGE_SCORE_SCHEMA'], + ['qa-health-rubric', 'buildQaHealthRubricJudgePrompt(section)', 'JUDGE_SCORE_SCHEMA'], + ['qa-anti-refusal', 'buildQaAntiRefusalJudgePrompt(diffAwareSection, rulesSection)', 'QA_ANTI_REFUSAL_JUDGE_SCHEMA'], + ['cross-skill', 'buildCrossSkillConsistencyJudgePrompt(collected)', 'CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA'], + ['voice', 'buildVoiceDirectiveJudgePrompt(voiceSection)', 'VOICE_DIRECTIVE_JUDGE_SCHEMA'], + ]; + + test.each(wiring)('%s sends its schema only when its complete calibration landed', (id, call, schema) => { + const recorded = result(id); + expect(recorded.complete).toBe(true); + const sent = evalSource.includes(`${call}, undefined, { jsonSchema: ${schema} })`); + expect(evalSource.includes(`${call})`) || sent).toBe(true); + expect(sent).toBe(recorded.landing.lands); + }); + + test('armJudge sends ARM_JUDGE_SCHEMA because its calibration landed', () => { + expect(result('arm').landing.lands).toBe(true); + expect(judgeSource).toContain('buildArmJudgePrompt(task, diff), ARM_JUDGE_MODEL, { jsonSchema: ARM_JUDGE_SCHEMA })'); + }); + + test('uncalibrated workflow configurations stay on today\'s request; only ship keeps its existing schema', () => { + expect(fs.existsSync(path.join(import.meta.dir, 'fixtures', 'judge-calibration', 'workflow-default', 'runs'))).toBe(false); + expect(evalSource.match(/schemaTransport: true/g)).toHaveLength(1); + expect(evalSource.match(/compactReasoning: true/g)).toHaveLength(1); + }); +});