test(judge-calibration): pin the cookie calibration's prompt and recorded result (GSTA-91)

/ship Step 7 coverage audit:
- the cookie-workflow calibration builds exactly the prompt the cookie eval sends
  for the same bundle;
- analyze() reproduces the committed comparison-1 result from its labels,
  samples, stop record and request options (request_hash covers model and
  options only, so prompt text and bundle bytes are not pinned).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-26 01:14:02 +00:00
1 parent 171d7a92a2
commit ce9c41c6d4
1 file changed
+28 -1
+28 -1
View File
@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { callJudge, JudgeRefusalError, type CallJudgeOptions, type JudgeResponseMeta } from './helpers/llm-judge';
import { COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input';
import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input';
import {
analyze, classifyError, loadCorpus, undispatched, PANEL_SAMPLES, PHASES, planCalls, priceManifest, runCalls, runSample,
type CalibrationConfig, type CorpusItem, type JudgeCall, type SampleRecord,
@@ -233,6 +233,18 @@ describe('committed calibration configurations and corpora', () => {
}
});
// /ship Step 7 coverage audit (GSTA-91).
// Value: protects=the cookie-workflow calibration sends the eval's exact prompt for the same bundle;
// fails_when=its build drops COOKIE_WORKFLOW_JUDGE context/goal or adds a capability the eval does not send;
// why_new=the corpus test only checks the bundle is contained in the prompt; seam=none;
// no_claim=provider behavior (no request is sent; both prompts come from the live SKILL.md and BROWSER.md)
test('the cookie-workflow calibration builds the prompt the cookie eval sends', async () => {
const { CALIBRATION_CONFIGS } = await import('../scripts/judge-calibration-configs');
const cookie = CALIBRATION_CONFIGS.find(config => config.id === 'cookie-workflow')!;
const production = buildCookieWorkflowJudgeInput(path.join(import.meta.dir, '..'));
expect(cookie.build({ bundle: production.text })).toBe(production.prompt);
});
test('panel verdicts restate each eval pass rule', async () => {
const { CALIBRATION_CONFIGS } = await import('../scripts/judge-calibration-configs');
const byId = Object.fromEntries(CALIBRATION_CONFIGS.map(config => [config.id, config]));
@@ -284,6 +296,21 @@ describe('landed schemas match the recorded calibration', () => {
expect(sent).toBe(recorded.landing.lands);
});
// /ship Step 7 coverage audit (GSTA-91).
// Value: protects=the held cookie-workflow result is what analyze() derives from its committed labels, samples, stop record and request options;
// fails_when=result.json is hand-edited, a corpus label moves after the run, a sample is altered, or the config's model/options drift;
// why_new=no test reads samples.jsonl or stop.json, and the wiring test trusts result.json's landing verbatim; seam=none;
// no_claim=that the recorded samples are genuine provider output (the committed JSONL stands in for the paid run), or that prompt
// text and bundle bytes are unchanged since the run (request_hash covers model and options only)
test('the recorded cookie-workflow result is reproducible from its committed labels, samples, stop record and request options', async () => {
const { CALIBRATION_CONFIGS } = await import('../scripts/judge-calibration-configs');
const cookie = CALIBRATION_CONFIGS.find(config => config.id === 'cookie-workflow')!;
const runs = path.join(import.meta.dir, 'fixtures', 'judge-calibration', 'cookie-workflow', 'runs');
const samples: SampleRecord[] = fs.readFileSync(path.join(runs, 'comparison-1.samples.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line));
const { stopped } = JSON.parse(fs.readFileSync(path.join(runs, 'comparison-1.stop.json'), 'utf8'));
expect(JSON.parse(JSON.stringify(analyze(cookie, loadCorpus('cookie-workflow'), samples, stopped)))).toEqual(result('cookie-workflow'));
});
test('armJudge sends ARM_JUDGE_SCHEMA because its calibration landed', () => {
expect(result('arm').landing.lands).toBe(true);
expect(judgeSource).toContain('buildArmJudgePrompt(task, diff), ARM_JUDGE_MODEL, { jsonSchema: ARM_JUDGE_SCHEMA })');