eval policy v3: judge panels gate on the per-dimension median of 3 samples

A judge dimension now passes when at least 2 of its 3 samples meet the
unchanged threshold (judgePanelMedian); any other sample count or a
non-numeric sample fails closed. Thresholds, sample count, judge model and
prompts are unchanged; panel logs show every sample and the mean, and name
the median as the gate. Applies to the skill-llm-eval panels, the
workflow-judge cache, the auq-matrix substance panel and the calibration
replicas. EVAL_POLICY 2 -> 3 and HARNESS_VERSION 1 -> 2 (verdicts change),
so every case starts a new pass-rate series. Replays the captured panels
from jobs 112032744416 and 112049856586 as replayed evidence.
This commit is contained in:
garrytan committed 2026-10-06 17:43:29 +00:00
1 parent a5450908e7
commit e7c11039ea
28 files changed
+1141 -139

No files matched your search

+5 -5
View File
@@ -17,16 +17,16 @@ const grader = (scores: number[]) => {
};
describe('auq-matrix substance panel (C4)', () => {
test('one capture is graded by a full panel and gates on the mean at the unchanged minimum', async () => {
test('one capture is graded by a full panel and gates on the median at the unchanged minimum', async () => {
expect(AUQ_SUBSTANCE_MIN).toBe(4);
const { grade, seen } = grader([5, 3, 4]);
const panel = await auqSubstancePanel('Recommendation: A because B', grade);
expect(seen).toEqual(Array(JUDGE_PANEL_SAMPLES).fill('Recommendation: A because B'));
expect(panel).toMatchObject({ mean: 4, passed: true });
expect(panel).toMatchObject({ median: 4, mean: 4, passed: true });
});
test('a single low sample no longer decides the case, but a low mean still fails', async () => {
expect((await auqSubstancePanel('q', grader([4, 3, 5]).grade)).passed).toBe(true);
expect((await auqSubstancePanel('q', grader([4, 3, 4]).grade))).toMatchObject({ passed: false });
test('two of three samples at the minimum pass; two below it fail; the mean is only reported', async () => {
expect((await auqSubstancePanel('q', grader([4, 3, 4]).grade))).toMatchObject({ median: 4, passed: true });
expect((await auqSubstancePanel('q', grader([5, 3, 3]).grade))).toMatchObject({ median: 3, passed: false });
expect((await auqSubstancePanel('q', grader([3, 3, 3]).grade)).mean).toBe(3);
});
test('a failed sample fails the panel without resampling', async () => {
+3 -3
View File
@@ -11,7 +11,7 @@ import { selectTests } from './helpers/test-selection';
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
import { selectPrProfile } from '../scripts/test-pr-profile';
import { JUDGE_MS } from './helpers/eval-budgets';
import { JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
import { JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
import { COOKIE_MANUAL_REVIEW_FILE, getCookieWorkflowManualReview, isManualReviewEntry } from './helpers/cookie-workflow-manual-review';
const ROOT = resolve(import.meta.dir, '..');
@@ -70,7 +70,7 @@ function actualCookieCallback(root: string, overrides: {
const records: EvalTestEntry[] = [];
const attempts = new Map<string, { attempt: number }>();
let callback: () => Promise<void> = async () => { throw new Error('Judge callback was not registered'); };
new Function('describeIfSelected', 'testIfSelected', 'ROOT', 'buildCookieWorkflowJudgeInput', 'resolveEvalModel', 'callJudge', 'COOKIE_WORKFLOW_JUDGE', 'JUDGE_MS', 'WORKFLOW_JUDGE_TEST_MS', 'WORKFLOW_JUDGE_RECORD_MS', 'evalCollector', 'expect', 'console', 'readWorkflowJudgeInput', 'buildWorkflowJudgePrompt', 'prepareWorkflowJudgeCache', 'workflowJudgeAttempts', 'performance', 'setTimeout', 'clearTimeout', 'JudgeRefusalError', 'getCookieWorkflowManualReview', 'DEFAULT_JUDGE_MAX_TOKENS', 'WORKFLOW_JUDGE_RESPONSE_SCHEMA', 'validWorkflowJudgeScore', 'judgePanel', 'judgePanelMean', 'judgePanelReasoning', 'JUDGE_SCORE_DIMENSIONS', registration)(
new Function('describeIfSelected', 'testIfSelected', 'ROOT', 'buildCookieWorkflowJudgeInput', 'resolveEvalModel', 'callJudge', 'COOKIE_WORKFLOW_JUDGE', 'JUDGE_MS', 'WORKFLOW_JUDGE_TEST_MS', 'WORKFLOW_JUDGE_RECORD_MS', 'evalCollector', 'expect', 'console', 'readWorkflowJudgeInput', 'buildWorkflowJudgePrompt', 'prepareWorkflowJudgeCache', 'workflowJudgeAttempts', 'performance', 'setTimeout', 'clearTimeout', 'JudgeRefusalError', 'getCookieWorkflowManualReview', 'DEFAULT_JUDGE_MAX_TOKENS', 'WORKFLOW_JUDGE_RESPONSE_SCHEMA', 'validWorkflowJudgeScore', 'judgePanel', 'judgePanelMean', 'judgePanelMedian', 'judgePanelReasoning', 'JUDGE_SCORE_DIMENSIONS', registration)(
(_suite: string, names: string[], run: () => void) => { expect(names).toEqual([NAME]); run(); },
(name: string, run: () => Promise<void>, budget: number) => { expect(name).toBe(NAME); expect(budget).toBe(JUDGE_MS + 10_000); callback = run; },
root, buildCookieWorkflowJudgeInput, (_kind: string, explicit?: string) => explicit ?? 'fixture-model',
@@ -85,7 +85,7 @@ function actualCookieCallback(root: string, overrides: {
attempts, overrides.clock ? { now: overrides.clock } : performance,
overrides.setTimer ?? setTimeout, overrides.clearTimer ?? clearTimeout,
JudgeRefusalError, getCookieWorkflowManualReview, DEFAULT_JUDGE_MAX_TOKENS,
WORKFLOW_JUDGE_RESPONSE_SCHEMA, validWorkflowJudgeScore, judgePanel, judgePanelMean, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS,
WORKFLOW_JUDGE_RESPONSE_SCHEMA, validWorkflowJudgeScore, judgePanel, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS,
);
return { run: () => callback(), requests, records, attempts };
}
+893
View File
@@ -0,0 +1,893 @@
{
"provenance": "Replayed evidence, not a CI pass: every judge panel printed by eval-slices (2) jobs 112032744416 (run 37389946212, head 1536f7f6b) and 112049856586 (run 37395241262, head a5450908e). Only the numeric and boolean sample fields the gates read are kept; reasoning text is omitted.",
"panels": [
{
"job": "112032744416",
"case": "Setup block",
"samples": [
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 3
}
]
},
{
"job": "112032744416",
"case": "Browse SKILL.md",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "QA health rubric",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "QA anti-refusal",
"samples": [
{
"would_browse": true,
"confidence": 5
},
{
"would_browse": true,
"confidence": 5
},
{
"would_browse": true,
"confidence": 5
}
]
},
{
"job": "112032744416",
"case": "QA workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 3,
"completeness": 4,
"actionability": 3
},
{
"clarity": 3,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "Cross-skill consistency",
"samples": [
{
"consistent": true,
"score": 4
},
{
"consistent": true,
"score": 4
},
{
"consistent": true,
"score": 4
}
]
},
{
"job": "112032744416",
"case": "document-release/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "ship/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "plan-ceo-review/SKILL.md modes",
"samples": [
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "plan-design-review/SKILL.md passes",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "plan-eng-review/SKILL.md sections",
"samples": [
{
"clarity": 3,
"completeness": 4,
"actionability": 3
},
{
"clarity": 3,
"completeness": 5,
"actionability": 4
},
{
"clarity": 3,
"completeness": 4,
"actionability": 3
}
]
},
{
"job": "112032744416",
"case": "design-review/SKILL.md fix loop",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "design-consultation/SKILL.md research",
"samples": [
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "land-and-deploy/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "canary/SKILL.md monitoring loop",
"samples": [
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "benchmark/SKILL.md perf collection",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "setup-deploy/SKILL.md platform setup",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "sync-gbrain/SKILL.md read-only readiness",
"samples": [
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "retro/SKILL.md instructions",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "review/SKILL.md workflow",
"samples": [
{
"clarity": 3,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "qa-only/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "gstack-upgrade/SKILL.md upgrade flow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112032744416",
"case": "Voice directive",
"samples": [
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
},
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
},
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
}
]
},
{
"job": "112032744416",
"case": "setup-browser-cookies/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "document-release/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "ship/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "plan-ceo-review/SKILL.md modes",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 3,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "plan-eng-review/SKILL.md sections",
"samples": [
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "design-review/SKILL.md fix loop",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "plan-design-review/SKILL.md passes",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "design-consultation/SKILL.md research",
"samples": [
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 4,
"completeness": 5,
"actionability": 4
},
{
"clarity": 3,
"completeness": 4,
"actionability": 3
}
]
},
{
"job": "112049856586",
"case": "land-and-deploy/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "canary/SKILL.md monitoring loop",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 3,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "setup-deploy/SKILL.md platform setup",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "benchmark/SKILL.md perf collection",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "sync-gbrain/SKILL.md read-only readiness",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 3,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "retro/SKILL.md instructions",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "review/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "qa-only/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "gstack-upgrade/SKILL.md upgrade flow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
},
{
"job": "112049856586",
"case": "Voice directive",
"samples": [
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
},
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
},
{
"directness": 5,
"concreteness": 5,
"avoids_corporate": 5,
"avoids_ai_vocabulary": 4,
"connects_user_outcomes": 5
}
]
},
{
"job": "112049856586",
"case": "setup-browser-cookies/SKILL.md workflow",
"samples": [
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
},
{
"clarity": 4,
"completeness": 4,
"actionability": 4
}
]
}
]
}
+1 -1
View File
@@ -131,7 +131,7 @@ describe('v2 writer', () => {
expect(errors).toEqual([]);
const expected = caseSeriesIdentitiesV2([id], treeEntries(ROOT))[id]!;
expect(records[0]).toMatchObject({ series_identity: expected.identity, series_fingerprint: expected.fingerprint, harness_version: HARNESS_VERSION,
policy_version: 2 });
policy_version: EVAL_POLICY.version });
fs.rmSync(dir, { recursive: true, force: true });
});
});
+6 -5
View File
@@ -1,11 +1,12 @@
/**
* auq-matrix recommendation substance (C4, approved 2026-10-04): one native
* capture per skill, scored by a JUDGE_PANEL_SAMPLES panel of the same grader
* on that capture. The panel mean gates against the unchanged minimum; a sample
* error fails the panel and is never resampled (judgePanel).
* on that capture. The panel median (2 of 3 samples) gates against the unchanged
* minimum and the mean is reported; a sample error fails the panel and is never
* resampled (judgePanel).
*/
import { gradeAuqRecommendation } from './auq-sdk-capture';
import { judgePanel, judgePanelMean } from './llm-judge';
import { judgePanel, judgePanelSummary } from './llm-judge';
export const AUQ_SUBSTANCE_MIN = 4;
@@ -13,6 +14,6 @@ type Grade = Awaited<ReturnType<typeof gradeAuqRecommendation>>;
export async function auqSubstancePanel(text: string, grade: (text: string) => Promise<Grade> = gradeAuqRecommendation) {
const samples = await judgePanel(() => grade(text));
const mean = judgePanelMean(samples, ['substance']).substance;
return { samples, mean, passed: mean >= AUQ_SUBSTANCE_MIN };
const { median, mean } = judgePanelSummary(samples, ['substance']);
return { samples, median: median.substance, mean: mean.substance, passed: median.substance >= AUQ_SUBSTANCE_MIN };
}
+1 -1
View File
@@ -306,7 +306,7 @@ const detailPart = (text: string): string => {
/**
* Detail the stored error line cannot carry (it keeps only the matcher header).
* A judge record with a `>= N` threshold names each dimension below it, its
* mean, its sample count and the first sample sentence naming that dimension;
* gating value (the panel median since EVAL_POLICY v3, in the `mean` field), its sample count and the first sample sentence naming that dimension;
* otherwise a Bun matcher message yields its Expected/Received values.
*/
export function failureDetailOf(text: string | undefined,
+1
View File
@@ -120,6 +120,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/impeccable-detect-dom-sample.json': ['test/impeccable-fixtures.test.ts'],
'test/fixtures/impeccable-detect-help.txt': ['test/impeccable-fixtures.test.ts'],
'test/fixtures/judge-calibration/**': ['scripts/judge-calibration.ts', 'test/judge-calibration.test.ts'],
'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'],
'test/fixtures/module-size-ratchet.json': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
'test/fixtures/module-size/**': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
'test/fixtures/native-auq-boxed-full-body-sep21.json': ['test/auq-native-capture.test.ts'],
+26 -3
View File
@@ -228,8 +228,10 @@ export const JUDGE_PANEL_SAMPLES = 3;
/**
* Judge panel (EVAL_POLICY.judge): every `judge`-kind entry draws a fixed number of
* independent samples of the SAME prompt concurrently, inside its unchanged
* JUDGE_MS budget. Numeric dimensions gate on the per-dimension panel mean
* against the unchanged minimum; boolean fields gate on a strict majority.
* JUDGE_MS budget. Numeric dimensions gate on the per-dimension median of the
* three samples (at least 2 of 3 at or above the unchanged minimum,
* judgePanelMedian); the mean is reported for information only. Boolean
* fields gate on a strict majority.
* A sample that errors (refusal, truncation, non-JSON, malformed field) fails
* the whole panel and is never resampled. callJudge's 429 backoff happens
* before any model output exists, so it is transport, not a verdict retry.
@@ -247,7 +249,23 @@ export async function judgePanel<T>(sample: () => Promise<T>): Promise<T[]> {
throw first.reason;
}
/** Per-dimension mean over a panel; any non-finite sample value fails the panel. */
/**
* The judge gate (EVAL_POLICY v3): per-dimension median of exactly
* JUDGE_PANEL_SAMPLES samples, so a dimension passes when at least 2 of 3
* samples meet its unchanged threshold. Any other sample count or a
* non-finite value fails the panel closed.
*/
export function judgePanelMedian<K extends string>(samples: ReadonlyArray<Record<K, unknown>>, keys: readonly K[]): Record<K, number> {
if (samples.length !== JUDGE_PANEL_SAMPLES) throw new Error(`Judge panel needs exactly ${JUDGE_PANEL_SAMPLES} samples, got ${samples.length}`);
return Object.fromEntries(keys.map(key => {
const values = samples.map(sample => sample && typeof sample === 'object' ? sample[key] : undefined);
const bad = values.findIndex(value => typeof value !== 'number' || !Number.isFinite(value));
if (bad !== -1) throw new Error(`Judge panel sample ${bad + 1} has non-numeric ${key}: ${JSON.stringify(values[bad])}`);
return [key, [...(values as number[])].sort((a, b) => a - b)[1]!];
})) as Record<K, number>;
}
/** Per-dimension mean over a panel, reported beside the median gate; any non-finite sample value fails the panel. */
export function judgePanelMean<K extends string>(samples: ReadonlyArray<Record<K, unknown>>, keys: readonly K[]): Record<K, number> {
if (samples.length === 0) throw new Error('Judge panel has no samples');
return Object.fromEntries(keys.map(key => {
@@ -258,6 +276,11 @@ export function judgePanelMean<K extends string>(samples: ReadonlyArray<Record<K
})) as Record<K, number>;
}
/** The median gate plus the informational mean, for panel reports. */
export function judgePanelSummary<K extends string>(samples: ReadonlyArray<Record<K, unknown>>, keys: readonly K[]): { median: Record<K, number>; mean: Record<K, number> } {
return { median: judgePanelMedian(samples, keys), mean: judgePanelMean(samples, keys) };
}
/** Strict majority of a boolean field; any non-boolean sample value fails the panel. */
export function judgePanelMajority<K extends string>(samples: ReadonlyArray<Record<K, unknown>>, key: K): boolean {
if (samples.length === 0) throw new Error('Judge panel has no samples');
+14 -7
View File
@@ -42,7 +42,8 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
/**
* Paid-eval verdict policy, pre-registered (v1 approved 2026-09-29; v2
* approved 2026-10-04, D1 of docs/test-audit-2026-10.md). Frozen before the
* approved 2026-10-04, D1 of docs/test-audit-2026-10.md; v3 approved
* 2026-10-06, the judge median gate). Frozen before the
* census: any change after seeing census results needs Garry's re-approval
* and a fresh census, and bumps `version` (every trial record carries it as
* policy_version; readers score only their own version, so pass-rate history
@@ -51,7 +52,11 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
* fixtures, its skills' prompt files) + HARNESS_VERSION
* (scripts/harness-version.json), keyed with model and CLI
* version; weekly history = scheduled main runs plus main
* dispatches; only v2 trials count toward quarantine entry.
* dispatches; only current-version trials count toward
* quarantine entry.
* v3 - the judge gate changes from the panel mean to the
* per-dimension median of 3 samples; v2's identity and
* history rules are unchanged, and v2 trials stop scoring.
* panel - behavior cases and quarantined cases run n independent
* trials; a behavior panel PASSES at >= k passing trials with
* no contract violation. Rule and judge cases run one trial.
@@ -59,10 +64,12 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
* new-policy trials; exit at >= `exit.rate` over >=
* `exit.minTrials`; at most `capFraction` of each tier's
* blocking cases; an entry expires after `expiryWeeklyRuns`.
* judge - a judge case draws `samples` independent samples of one
* prompt concurrently; numeric dimensions gate on the panel
* mean against the unchanged threshold, booleans on a strict
* majority; an erroring sample fails the panel, never resampled.
* judge - a judge case draws exactly `samples` (3) independent samples
* of one prompt concurrently; a numeric dimension passes when
* its median (at least 2 of 3 samples) meets the unchanged
* threshold, and the mean is reported only; booleans gate on a
* strict majority; any other sample count or an erroring sample
* fails the panel, never resampled.
* drift - one-sided Fisher exact alarm between input-identity series
* (Holm-controlled across the cases tested in one report).
* infraRedispatch - a census whose every red verdict is machine-classified
@@ -70,7 +77,7 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
* a new run; both runs are reported.
*/
export const EVAL_POLICY = {
version: 2,
version: 3,
panel: { n: 3, k: 2 },
quarantine: {
entry: { rate: 0.95, minTrials: 10 },
+8 -7
View File
@@ -4,7 +4,7 @@ import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
import { JUDGE_MS } from './eval-budgets';
import { JUDGE_PANEL_SAMPLES, JUDGE_SCORE_DIMENSIONS, judgePanelMean, type JudgeScore } from './llm-judge';
import { JUDGE_PANEL_SAMPLES, JUDGE_SCORE_DIMENSIONS, judgePanelMedian, type JudgeScore } from './llm-judge';
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA, WORKFLOW_JUDGE_REASONING_WORD_LIMIT } from './workflow-judge-input';
import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, storeEvalInputCache,
type EvalCacheValue, type EvalInputIdentity, type EvalPassingProof } from '../../scripts/eval-input-cache';
@@ -49,13 +49,13 @@ export function validWorkflowJudgeScore(value: EvalCacheValue, thresholds: Thres
const SAMPLE_RANGE: Thresholds = { clarity: 1, completeness: 1, actionability: 1 };
/** A complete judge panel: exactly JUDGE_PANEL_SAMPLES of valid samples whose per-dimension mean meets every threshold. */
/** A complete judge panel: exactly JUDGE_PANEL_SAMPLES of valid samples whose per-dimension median (2 of 3) meets every threshold. */
export function validWorkflowJudgePanel(value: EvalCacheValue, thresholds: Thresholds, compactReasoning = false): value is { samples: Array<JudgeScore & EvalCacheValue> } {
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).join(',') !== 'samples'
|| !Array.isArray(value.samples) || value.samples.length !== JUDGE_PANEL_SAMPLES
|| !value.samples.every(sample => validWorkflowJudgeScore(sample, SAMPLE_RANGE, compactReasoning))) return false;
const mean = judgePanelMean(value.samples as JudgeScore[], JUDGE_SCORE_DIMENSIONS);
return JUDGE_SCORE_DIMENSIONS.every(key => mean[key] >= thresholds[key]);
const median = judgePanelMedian(value.samples as JudgeScore[], JUDGE_SCORE_DIMENSIONS);
return JUDGE_SCORE_DIMENSIONS.every(key => median[key] >= thresholds[key]);
}
export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): {
@@ -140,9 +140,10 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): {
}
/**
* Pass floors for the browse reference judge panel mean. The stored baseline
* is recorded for comparison only: a three-sample mean is too noisy for a
* no-dip ratchet, and gating on it silently raised the clarity floor to 4.
* Pass floors for the browse reference judge panel, applied to its gating
* per-dimension median (EVAL_POLICY v3). The stored baseline is recorded for
* comparison only: a three-sample aggregate is too noisy for a no-dip ratchet,
* and gating on it silently raised the clarity floor to 4.
*/
export const BROWSE_JUDGE_FLOORS = { clarity: 3, completeness: 4, actionability: 4 } as const;
+6 -3
View File
@@ -238,7 +238,8 @@ describe('committed calibration configurations and corpora', () => {
const qa = (clarity: number, completeness: number, actionability: number) => ({ clarity, completeness, actionability, reasoning: '' });
for (const id of ['qa-workflow', 'qa-health-rubric', 'workflow-default']) {
expect(byId[id]!.verdict([qa(3, 3, 4), qa(3, 3, 4), qa(3, 3, 4)])).toBe('pass');
expect(byId[id]!.verdict([qa(5, 5, 4), qa(5, 5, 4), qa(5, 5, 3)])).toBe('fail');
expect(byId[id]!.verdict([qa(5, 5, 4), qa(5, 5, 4), qa(5, 5, 3)])).toBe('pass');
expect(byId[id]!.verdict([qa(5, 5, 4), qa(5, 5, 3), qa(5, 5, 3)])).toBe('fail');
expect(() => byId[id]!.validate({ ...qa(3, 3, 4), clarity: '3' })).toThrow();
}
const arm = (over_engineering: number) => ({ over_engineering, construct: over_engineering ? 'helper' : 'none', reasoning: '' });
@@ -247,10 +248,12 @@ describe('committed calibration configurations and corpora', () => {
expect(() => byId.arm!.validate({ over_engineering: 0, construct: 'helper', reasoning: '' })).toThrow();
const browse = (would_browse: boolean, confidence: number) => ({ would_browse, confidence, fallback_behavior: '', reasoning: '' });
expect(byId['qa-anti-refusal']!.verdict([browse(true, 4), browse(true, 4), browse(false, 5)])).toBe('pass');
expect(byId['qa-anti-refusal']!.verdict([browse(true, 4), browse(true, 3), browse(true, 4)])).toBe('fail');
expect(byId['qa-anti-refusal']!.verdict([browse(true, 4), browse(true, 3), browse(true, 4)])).toBe('pass');
expect(byId['qa-anti-refusal']!.verdict([browse(true, 4), browse(true, 3), browse(true, 3)])).toBe('fail');
const voice = (low: number) => ({ directness: 5, concreteness: 5, avoids_corporate: 5, avoids_ai_vocabulary: low, connects_user_outcomes: 5, reasoning: '' });
expect(byId.voice!.verdict([voice(4), voice(4), voice(4)])).toBe('pass');
expect(byId.voice!.verdict([voice(4), voice(4), voice(3)])).toBe('fail');
expect(byId.voice!.verdict([voice(4), voice(4), voice(3)])).toBe('pass');
expect(byId.voice!.verdict([voice(4), voice(3), voice(3)])).toBe('fail');
});
});
+62
View File
@@ -0,0 +1,62 @@
import { describe, expect, test } from 'bun:test';
import replay from './fixtures/judge-panel-replay-2026-10-05.json';
import { judgePanelMajority, judgePanelMean, judgePanelMedian, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
import { BROWSE_JUDGE_FLOORS } from './helpers/workflow-judge-cache';
import { EVAL_POLICY } from './helpers/periodic-exclude-data';
const gate = (scores: number[], threshold = 4) => judgePanelMedian(scores.map(score => ({ score })), ['score']).score >= threshold;
describe('judge gate: per-dimension median of exactly three samples (EVAL_POLICY v3)', () => {
test('a dimension passes when at least 2 of 3 samples meet the unchanged threshold', () => {
expect(EVAL_POLICY.judge.samples).toBe(JUDGE_PANEL_SAMPLES);
expect(gate([4, 4, 3])).toBe(true);
expect(gate([4, 3, 3])).toBe(false);
expect(gate([5, 5, 3])).toBe(true);
expect(gate([5, 3, 3])).toBe(false);
expect(gate([3, 4, 4])).toBe(true);
});
test('anything but exactly three valid samples fails closed', () => {
expect(() => gate([4, 4])).toThrow('exactly 3 samples');
expect(() => gate([4, 4, 4, 4])).toThrow('exactly 3 samples');
expect(() => judgePanelMedian([{ score: 4 }, { score: '4' }, { score: 4 }] as any, ['score'])).toThrow('sample 2 has non-numeric score');
expect(() => judgePanelMedian([{ score: 4 }, null, { score: 4 }] as any, ['score'])).toThrow('sample 2');
expect(() => judgePanelMedian([{ score: 4 }, { score: Number.NaN }, { score: 4 }], ['score'])).toThrow('non-numeric');
});
test('each dimension is gated on its own: one failing dimension fails the panel', () => {
const samples = [{ clarity: 5, actionability: 3 }, { clarity: 5, actionability: 3 }, { clarity: 5, actionability: 4 }];
const median = judgePanelMedian(samples, ['clarity', 'actionability']);
expect(median).toEqual({ clarity: 5, actionability: 3 });
expect(median.clarity >= 4 && median.actionability >= 4).toBe(false);
});
});
type Sample = Record<string, number | boolean>;
const SCORE = ['clarity', 'completeness', 'actionability'] as const;
function verdict(name: string, samples: Sample[], aggregate: typeof judgePanelMean): boolean {
const meets = (keys: readonly string[], floors: Record<string, number>) => {
const scores = aggregate(samples as Record<string, unknown>[], keys);
return keys.every(key => scores[key]! >= floors[key]!);
};
if (name === 'Setup block') return meets(['clarity', 'actionability'], { clarity: 3, actionability: 3 });
if (name === 'Browse SKILL.md') return meets(SCORE, BROWSE_JUDGE_FLOORS);
if (name === 'QA anti-refusal') return judgePanelMajority(samples as Record<string, unknown>[], 'would_browse') && meets(['confidence'], { confidence: 4 });
if (name === 'Cross-skill consistency') return judgePanelMajority(samples as Record<string, unknown>[], 'consistent') && meets(['score'], { score: 4 });
if (name === 'Voice directive') {
const keys = ['directness', 'concreteness', 'avoids_corporate', 'avoids_ai_vocabulary', 'connects_user_outcomes'];
return meets(keys, Object.fromEntries(keys.map(key => [key, 4])));
}
return meets(SCORE, { clarity: 3, completeness: 3, actionability: 4 });
}
test('replayed panels from jobs 112032744416 and 112049856586 (replayed evidence, not a CI pass)', () => {
const rows = replay.panels.map(panel => ({ job: panel.job, case: panel.case,
mean: verdict(panel.case, panel.samples as Sample[], judgePanelMean), median: verdict(panel.case, panel.samples as Sample[], judgePanelMedian) }));
expect(rows).toHaveLength(42);
expect(rows.filter(row => !row.mean).map(row => `${row.job} ${row.case}`)).toEqual([
'112032744416 QA workflow', '112032744416 plan-eng-review/SKILL.md sections', '112049856586 design-consultation/SKILL.md research']);
expect(rows.filter(row => !row.median).map(row => `${row.job} ${row.case}`)).toEqual(['112032744416 plan-eng-review/SKILL.md sections']);
// Exactly these two panels change verdict: each had 2 of 3 samples at every threshold.
expect(rows.filter(row => row.median && !row.mean).map(row => row.case)).toEqual(['QA workflow', 'design-consultation/SKILL.md research']);
});
+3 -3
View File
@@ -7,7 +7,7 @@
* against the same threshold the production E2E tests use (>= 4).
*
* Each fixture is a pre-registered 3-sample judge panel: numeric substance
* gates on the panel mean, the boolean checks on a 2-of-3 majority, and an
* gates on the panel median (2 of 3 samples), the boolean checks on a 2-of-3 majority, and an
* erroring sample fails the panel (never resampled). Costs ~$0.12 per run.
* Touchfile-gated to test/helpers/llm-judge.ts so it fires on rubric
* tweaks but not every test run. Runs only under EVALS=1 with an API key.
@@ -15,7 +15,7 @@
import { expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { judgePanel, judgePanelMajority, judgePanelMean, judgePanelReasoning, judgeRecommendation } from './helpers/llm-judge';
import { judgePanel, judgePanelMajority, judgePanelMedian, judgePanelReasoning, judgeRecommendation } from './helpers/llm-judge';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
// Fixtures wrap a realistic AskUserQuestion shape so the judge sees the menu
@@ -45,7 +45,7 @@ async function judgeRecommendationPanel(text: string) {
present: judgePanelMajority(samples, 'present'),
commits: judgePanelMajority(samples, 'commits'),
has_because: judgePanelMajority(samples, 'has_because'),
reason_substance: judgePanelMean(samples, ['reason_substance']).reason_substance,
reason_substance: judgePanelMedian(samples, ['reason_substance']).reason_substance,
reasoning: judgePanelReasoning(samples),
};
}
+1 -1
View File
@@ -759,7 +759,7 @@ console.log("Ran 1 tests across 1 files. [1ms]"); process.exit(${fail ? 1 : 0});
expect(sliceExitCode(summary.outcomes)).toBe(0);
const env = JSON.parse(fs.readFileSync(path.join(evalDirBase, 'shards', shardSlug([key(3)]), 'env.json'), 'utf8'));
expect(env).toMatchObject({ GSTACK_EVAL_CASE_ID: 'review-sql-injection', GSTACK_EVAL_KIND: 'behavior', GSTACK_EVAL_TRIAL: '3',
GSTACK_EVAL_PANEL_N: '3', GSTACK_EVAL_PANEL_K: '2', GSTACK_EVAL_POLICY_VERSION: '2' });
GSTACK_EVAL_PANEL_N: '3', GSTACK_EVAL_PANEL_K: '2', GSTACK_EVAL_POLICY_VERSION: '3' });
expect(JSON.parse(env.EVALS_SELECTION_JSON).selected).toEqual(['review-sql-injection']);
} finally { fs.rmSync(evalDirBase, { recursive: true, force: true }); }
});
+2 -1
View File
@@ -77,8 +77,9 @@ describe('periodic exclude policy', () => {
describe('eval verdict policy (pre-registered)', () => {
test('EVAL_POLICY carries exactly the approved constants; a change needs re-approval and a version bump', () => {
// v2: approved 2026-10-04 (D1, docs/test-audit-2026-10.md); only identity and history changed.
// v3: approved 2026-10-06; the judge gate is the per-dimension median of 3 samples.
expect(EVAL_POLICY).toEqual({
version: 2,
version: 3,
panel: { n: 3, k: 2 },
quarantine: { entry: { rate: 0.95, minTrials: 10 }, exit: { rate: 0.97, minTrials: 10 }, capFraction: 0.10, expiryWeeklyRuns: 8 },
judge: { samples: 3 },
+5 -3
View File
@@ -141,12 +141,14 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
const text = capture.text;
const fmt = scoreAuqFormat(text);
let substance = 0;
let substanceMean = 0;
let samples: number[] = [];
let recPresent = false;
let hadBecause = false;
if (text.trim()) {
const panel = await auqSubstancePanel(text);
substance = panel.mean;
substance = panel.median;
substanceMean = panel.mean;
samples = panel.samples.map(g => g.substance);
recPresent = panel.samples[0].present;
hadBecause = panel.samples[0].hadLiteralBecause;
@@ -154,7 +156,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
// eslint-disable-next-line no-console
console.log(
`[AUQ-matrix ${m.skill}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
`missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance.toFixed(2)} [${samples.join(',')}] ` +
`missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance median=${substance} mean=${substanceMean.toFixed(2)} [${samples.join(',')}] ` +
`literalBecause=${hadBecause}`,
);
@@ -165,7 +167,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
// Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed.
problems.push(...auqMachineFormatProblems(capture.question));
if (substance < AUQ_SUBSTANCE_MIN) {
problems.push(`recommendation substance mean ${substance.toFixed(2)} < ${AUQ_SUBSTANCE_MIN} over samples [${samples.join(',')}] (boilerplate/weak)`);
problems.push(`recommendation substance median ${substance} < ${AUQ_SUBSTANCE_MIN} over samples [${samples.join(',')}] (boilerplate/weak)`);
}
if (problems.length > 0) {
throw new Error(
+17 -17
View File
@@ -14,7 +14,7 @@ import { afterAll, expect } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, VOICE_DIRECTIVE_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge';
import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, VOICE_DIRECTIVE_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMedian, judgePanelSummary, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge';
import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt';
import type { JudgeScore } from './helpers/llm-judge';
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input';
@@ -101,8 +101,8 @@ describeIfSelected('LLM-as-judge quality evals', [
const section = sliceBrowseSection('## Snapshot Flags');
const samples = await judgePanel(() => judge('browse skill reference (flags + commands)', section));
const scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
console.log('Browse SKILL.md panel:', JSON.stringify({ mean: scores, samples }, null, 2));
const { median: scores, mean } = judgePanelSummary(samples, JUDGE_SCORE_DIMENSIONS);
console.log('Browse SKILL.md panel:', JSON.stringify({ gate: 'median', median: scores, mean, samples }, null, 2));
const baselinesPath = path.join(ROOT, 'test', 'fixtures', 'eval-baselines.json');
const baselines = JSON.parse(fs.readFileSync(baselinesPath, 'utf-8'));
@@ -145,8 +145,8 @@ describeIfSelected('LLM-as-judge quality evals', [
const section = content.slice(setupStart, setupEnd);
const samples = await judgePanel(() => judge('setup/binary discovery instructions', section));
const scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
console.log('Setup block panel:', JSON.stringify({ mean: scores, samples }, null, 2));
const { median: scores, mean } = judgePanelSummary(samples, JUDGE_SCORE_DIMENSIONS);
console.log('Setup block panel:', JSON.stringify({ gate: 'median', median: scores, mean, samples }, null, 2));
evalCollector?.addTest({
name: 'setup block',
@@ -206,8 +206,8 @@ describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.
references: ['qa/templates/functional-report-template.md', 'qa/templates/qa-report-template.md', 'qa/references/issue-taxonomy.md'] }).text;
const samples = await judgePanel(() => callJudge<JudgeScore>(buildQaWorkflowJudgePrompt(section)));
const scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
console.log('QA workflow panel:', JSON.stringify({ mean: scores, samples }, null, 2));
const { median: scores, mean } = judgePanelSummary(samples, JUDGE_SCORE_DIMENSIONS);
console.log('QA workflow panel:', JSON.stringify({ gate: 'median', median: scores, mean, samples }, null, 2));
evalCollector?.addTest({
name: 'qa/SKILL.md workflow',
@@ -232,8 +232,8 @@ describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.
const section = sliceQaPatterns('## Health Score Rubric');
const samples = await judgePanel(() => callJudge<JudgeScore>(buildQaHealthRubricJudgePrompt(section), undefined, { jsonSchema: JUDGE_SCORE_SCHEMA }));
const scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
console.log('QA health rubric panel:', JSON.stringify({ mean: scores, samples }, null, 2));
const { median: scores, mean } = judgePanelSummary(samples, JUDGE_SCORE_DIMENSIONS);
console.log('QA health rubric panel:', JSON.stringify({ gate: 'median', median: scores, mean, samples }, null, 2));
evalCollector?.addTest({
name: 'qa/SKILL.md health rubric',
@@ -263,9 +263,9 @@ describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.
const rulesSection = sliceQaPatterns('## Important Rules');
const samples = await judgePanel(() => callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(buildQaAntiRefusalJudgePrompt(diffAwareSection, rulesSection), undefined, { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA }));
const result = { would_browse: judgePanelMajority(samples, 'would_browse'), ...judgePanelMean(samples, ['confidence'] as const) };
const result = { would_browse: judgePanelMajority(samples, 'would_browse'), ...judgePanelMedian(samples, ['confidence'] as const) };
console.log('QA anti-refusal panel:', JSON.stringify({ result, samples }, null, 2));
console.log('QA anti-refusal panel:', JSON.stringify({ gate: 'median', result, mean: judgePanelMean(samples, ['confidence'] as const), samples }, null, 2));
evalCollector?.addTest({
name: 'qa/SKILL.md anti-refusal',
@@ -308,9 +308,9 @@ describeIfSelected('Cross-skill consistency evals', ['cross-skill greptile consi
].join('\n\n');
const samples = await judgePanel(() => callJudge<{ consistent: boolean; issues: string[]; score: number; reasoning: string }>(buildCrossSkillConsistencyJudgePrompt(collected), undefined, { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA }));
const result = { consistent: judgePanelMajority(samples, 'consistent'), ...judgePanelMean(samples, ['score'] as const) };
const result = { consistent: judgePanelMajority(samples, 'consistent'), ...judgePanelMedian(samples, ['score'] as const) };
console.log('Cross-skill consistency panel:', JSON.stringify({ result, samples }, null, 2));
console.log('Cross-skill consistency panel:', JSON.stringify({ gate: 'median', result, mean: judgePanelMean(samples, ['score'] as const), samples }, null, 2));
evalCollector?.addTest({
name: 'cross-skill greptile consistency',
@@ -464,8 +464,8 @@ async function runWorkflowJudge(opts: {
if (opts.compactReasoning && !samples.every(sample => validWorkflowJudgeScore(sample as unknown as EvalCacheValue, { clarity: 1, completeness: 1, actionability: 1 }, true))) {
throw new Error('Workflow judge violated the compact response contract');
}
scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
console.log(`${opts.testName} panel:`, JSON.stringify({ mean: scores, samples }, null, 2));
scores = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);
console.log(`${opts.testName} panel:`, JSON.stringify({ gate: 'median', median: scores, mean: judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS), samples }, null, 2));
expect(scores.clarity).toBeGreaterThanOrEqual(thresholds.clarity);
expect(scores.completeness).toBeGreaterThanOrEqual(thresholds.completeness);
expect(scores.actionability).toBeGreaterThanOrEqual(thresholds.actionability);
@@ -752,9 +752,9 @@ describeIfSelected('Voice directive eval', ['voice directive tone'], () => {
connects_user_outcomes: number;
reasoning: string;
}>(buildVoiceDirectiveJudgePrompt(voiceSection), undefined, { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA }));
const result = judgePanelMean(samples, ['directness', 'concreteness', 'avoids_corporate', 'avoids_ai_vocabulary', 'connects_user_outcomes'] as const);
const { median: result, mean } = judgePanelSummary(samples, ['directness', 'concreteness', 'avoids_corporate', 'avoids_ai_vocabulary', 'connects_user_outcomes'] as const);
console.log('Voice directive panel:', JSON.stringify({ mean: result, samples }, null, 2));
console.log('Voice directive panel:', JSON.stringify({ gate: 'median', median: result, mean, samples }, null, 2));
evalCollector?.addTest({
name: 'voice directive tone',
+12 -12
View File
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
import { Messages } from '@anthropic-ai/sdk/resources/messages';
import { callJudge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMajority, judgePanelMean, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
import { callJudge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMajority, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
import { EVAL_POLICY } from './helpers/periodic-exclude-data';
import { getCookieWorkflowManualReview } from './helpers/cookie-workflow-manual-review';
import { resolveEvalModel } from '../lib/eval-model';
@@ -172,7 +172,7 @@ test('workflow registration preserves model work and reserves only terminal-reco
const body = source.split('async function runWorkflowJudge')[1]!.split('// Block 1:')[0]!;
const stages = ['workflowJudgeAttempts.set', 'readWorkflowJudgeInput(', 'cache.lookup()',
'judgePanel(() => callJudge<JudgeScore>(prompt, opts.model, { signal: controller.signal, max_tokens: maxTokens,',
'scores = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);',
'scores = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);',
'expect(scores.clarity)', 'expect(scores.completeness)', 'expect(scores.actionability)', 'cache.publish(samples, active)']
.map(stage => body.indexOf(stage));
expect(stages.every(position => position >= 0)).toBe(true);
@@ -207,7 +207,7 @@ function actualCallback(f: ReturnType<typeof fixture>, overrides: {
'evalCollector', 'expect', 'console', 'performance', 'JUDGE_MS', 'WORKFLOW_JUDGE_RECORD_MS',
'setTimeout', 'clearTimeout', 'JudgeRefusalError', 'getCookieWorkflowManualReview', 'DEFAULT_JUDGE_MAX_TOKENS', 'resolveEvalModel',
'WORKFLOW_JUDGE_RESPONSE_SCHEMA', 'validWorkflowJudgeScore',
'judgePanel', 'judgePanelMean', 'judgePanelReasoning', 'JUDGE_SCORE_DIMENSIONS',
'judgePanel', 'judgePanelMean', 'judgePanelMedian', 'judgePanelReasoning', 'JUDGE_SCORE_DIMENSIONS',
`${javascript}\nreturn runWorkflowJudge;`)(
f.root, overrides.read ?? readWorkflowJudgeInput, buildWorkflowJudgePrompt,
(options: WorkflowCacheOptions) => (overrides.prepare ?? prepareWorkflowJudgeCache)({ ...options, env: f.env }),
@@ -219,7 +219,7 @@ function actualCallback(f: ReturnType<typeof fixture>, overrides: {
overrides.setTimer ?? setTimeout, overrides.clearTimer ?? clearTimeout,
JudgeRefusalError, getCookieWorkflowManualReview, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel,
WORKFLOW_JUDGE_RESPONSE_SCHEMA, validWorkflowJudgeScore,
judgePanel, judgePanelMean, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS);
judgePanel, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS);
return { run, records, signals, prompts, attempts, options: { ...f.opts, suite: 'Cache regression' } };
}
@@ -572,14 +572,14 @@ describe('judge panel', () => {
expect(String(error)).toContain(`sample 1 of ${SAMPLES} failed beside scored samples`);
});
test('numeric dimensions gate on the per-dimension mean; one low sample can be outvoted, a low mean cannot', () => {
const outvoted = judgePanelMean([panelScore(2), panelScore(4), panelScore(4)], JUDGE_SCORE_DIMENSIONS);
expect(outvoted.clarity).toBeCloseTo(10 / 3);
expect(outvoted.clarity).toBeGreaterThanOrEqual(panelThresholds.clarity);
const low = judgePanelMean([panelScore(2), panelScore(2), panelScore(4)], JUDGE_SCORE_DIMENSIONS);
test('numeric dimensions gate on the per-dimension median: 2 of 3 samples decide, the mean is only reported', () => {
const outvoted = judgePanelMedian([panelScore(2), panelScore(4), panelScore(4)], JUDGE_SCORE_DIMENSIONS);
expect(outvoted.clarity).toBe(4);
expect(judgePanelMean([panelScore(2), panelScore(4), panelScore(4)], JUDGE_SCORE_DIMENSIONS).clarity).toBeCloseTo(10 / 3);
const low = judgePanelMedian([panelScore(2), panelScore(2), panelScore(4)], JUDGE_SCORE_DIMENSIONS);
expect(low.clarity).toBeLessThan(panelThresholds.clarity);
// No compensation across dimensions: each is averaged on its own.
expect(judgePanelMean([panelScore(5, 1), panelScore(5, 1), panelScore(5, 1)], JUDGE_SCORE_DIMENSIONS).completeness).toBe(1);
// No compensation across dimensions: each is gated on its own.
expect(judgePanelMedian([panelScore(5, 1), panelScore(5, 1), panelScore(5, 1)], JUDGE_SCORE_DIMENSIONS).completeness).toBe(1);
});
test('malformed sample fields fail the panel instead of averaging to NaN', () => {
@@ -621,7 +621,7 @@ describe('judge panel', () => {
});
});
// Stored browse reference panel means against the judge floors.
// Stored browse reference panel means (recorded before EVAL_POLICY v3) against the judge floors.
{
type Scores = { clarity: number; completeness: number; actionability: number };
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as