CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-09 18:28:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

eval policy v3: judge panels gate on the per-dimension median of 3 samples

Browse Source 
A judge dimension now passes when at least 2 of its 3 samples meet the
unchanged threshold (judgePanelMedian); any other sample count or a
non-numeric sample fails closed. Thresholds, sample count, judge model and
prompts are unchanged; panel logs show every sample and the mean, and name
the median as the gate. Applies to the skill-llm-eval panels, the
workflow-judge cache, the auq-matrix substance panel and the calibration
replicas. EVAL_POLICY 2 -> 3 and HARNESS_VERSION 1 -> 2 (verdicts change),
so every case starts a new pass-rate series. Replays the captured panels
from jobs 112032744416 and 112049856586 as replayed evidence.
This commit is contained in:
garrytan committed 2026-10-06 17:43:29 +00:00
1 parent a5450908e7
commit e7c11039ea
28 files changed
+1141 -139
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/harness-version.json
+40 -40
View File
@@ -1,5 +1,5 @@
{
"version": 1,
"version": 2,
"patterns": [
"test/helpers/agent-sdk-runner.ts",
"test/helpers/codex-session-runner.ts",
@@ -10,44 +10,44 @@
"test/helpers/llm-judge.ts"
],
"files": {
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":1,"nonBehavioral":"caseSelection runs single-case files whole (local --case diagnosis only; CI slices never take this path)"},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":1,"nonBehavioral":"--case CLI passes its own runner to runCaseDiagnosis (a self-import during top-level await deadlocked); CI slices never take this path"},
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/auq-substance-panel.ts": {"blob":"85d785f25fe6a24277d4009f9ccb7b1e05a270ef","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":1},
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":1},
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":1},
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/eval-store.ts": {"blob":"bc135ddcf13a45c9803995df2af1aff54a471e2d","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":1},
"test/helpers/llm-judge.ts": {"blob":"453a4c1d355f3811deca53f953767109a9ec9292","since":1},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":1},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":1},
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":1},
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":1},
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":1},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":1},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":1},
"test/helpers/pty/launch.ts": {"blob":"4ef0bcb7b5e7c35d2aac0cbffd7f8a303440b6fe","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":1},
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/pty/runners/floor.ts": {"blob":"465f01641fb3a18ae1c9d2613a9c2b26f8fb503e","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/pty/screen.ts": {"blob":"ee005f36ffa48617e6832ccbeb35e8e96873df9b","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":1},
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":1,"nonBehavioral":"reliability follow-ups integration before the first v2 census: session ledger, trial cause/cost fields and stream liveness are diagnostics; EVAL_POLICY v2 already starts fresh series"},
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":1},
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":1},
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":1}
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":2},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":2},
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":2},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":2},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":2},
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":2},
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":2},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":2},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":2},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":2},
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":2},
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":2},
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":2},
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":2},
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":2},
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":2},
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":2},
"test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":2},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":2},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":2},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":2},
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":2},
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":2},
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":2},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":2},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":2},
"test/helpers/pty/launch.ts": {"blob":"4ef0bcb7b5e7c35d2aac0cbffd7f8a303440b6fe","since":2},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":2},
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":2},
"test/helpers/pty/runners/floor.ts": {"blob":"465f01641fb3a18ae1c9d2613a9c2b26f8fb503e","since":2},
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":2},
"test/helpers/pty/screen.ts": {"blob":"ee005f36ffa48617e6832ccbeb35e8e96873df9b","since":2},
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":2},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":2},
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":2},
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":2},
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":2},
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":2},
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":2}
}
}
scripts/judge-calibration-configs.ts
+8 -8
View File
@@ -10,7 +10,7 @@ import {
buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS, JUDGE_SCORE_DIMENSIONS, DEFAULT_JUDGE_MAX_TOKENS,
judgePanelMean, judgePanelMajority,
judgePanelMedian, judgePanelMajority,
} from '../test/helpers/llm-judge';
import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
@@ -18,7 +18,7 @@ import type { CalibrationConfig, Verdict } from './judge-calibration';
type Sample = Record<string, unknown>;
/** What production rejects: judgePanelMean needs finite numbers, judgePanelMajority needs booleans. */
/** What production rejects: judgePanelMedian needs finite numbers, judgePanelMajority needs booleans. */
function requireFields(value: unknown, numbers: readonly string[], booleans: readonly string[] = []): void {
const sample = (value ?? {}) as Sample;
for (const key of numbers) {
@@ -29,13 +29,13 @@ function requireFields(value: unknown, numbers: readonly string[], booleans: rea
}
}
const meets = (mean: Record<string, number>, floors: Record<string, number>): Verdict =>
Object.entries(floors).every(([key, floor]) => mean[key]! >= floor) ? 'pass' : 'fail';
const meets = (median: Record<string, number>, floors: Record<string, number>): Verdict =>
Object.entries(floors).every(([key, floor]) => median[key]! >= floor) ? 'pass' : 'fail';
const QA_FLOORS = { clarity: 3, completeness: 3, actionability: 4 };
const scoreJudge = (floors: Record<string, number>) => ({
validate: (value: unknown) => requireFields(value, JUDGE_SCORE_DIMENSIONS),
verdict: (samples: unknown[]) => meets(judgePanelMean<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
verdict: (samples: unknown[]) => meets(judgePanelMedian<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
});
const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
@@ -80,7 +80,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
validate: value => requireFields(value, ['confidence'], ['would_browse']),
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'would_browse')
&& judgePanelMean<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
&& judgePanelMedian<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
estimatedOutputTokens: 1200,
},
{
@@ -89,7 +89,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
validate: value => requireFields(value, ['score'], ['consistent']),
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'consistent')
&& judgePanelMean<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
&& judgePanelMedian<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
estimatedOutputTokens: 1200,
},
{
@@ -97,7 +97,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
verdict: samples => meets(judgePanelMean<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
verdict: samples => meets(judgePanelMedian<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
Object.fromEntries(VOICE_DIRECTIVE_DIMENSIONS.map(key => [key, 4]))),
estimatedOutputTokens: 1200,
},
scripts/judge-controls.ts
+5 -4
View File
@@ -12,7 +12,7 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { buildQaWorkflowJudgePrompt, callJudge, judgePanelMean, DEFAULT_JUDGE_MAX_TOKENS, JUDGE_SCORE_DIMENSIONS, type CallJudgeOptions } from '../test/helpers/llm-judge';
import { buildQaWorkflowJudgePrompt, callJudge, judgePanelMean, judgePanelMedian, DEFAULT_JUDGE_MAX_TOKENS, JUDGE_SCORE_DIMENSIONS, type CallJudgeOptions } from '../test/helpers/llm-judge';
import { buildWorkflowJudgePrompt, readWorkflowJudgeInput, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
const ROOT = path.resolve(import.meta.dir, '..');
@@ -113,10 +113,11 @@ const results = await Promise.all(JUDGES.flatMap(judge => Object.entries(bundles
const prompt = judge.prompt(text);
const samples = await Promise.all(Array.from({ length: SAMPLES }, () => callJudge<Sample>(prompt, undefined, judge.options)));
const mean = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
const verdict = Object.entries(FLOORS).every(([key, floor]) => mean[key as keyof typeof FLOORS] >= floor) ? 'pass' : 'fail';
return { judge: judge.name, bundle, verdict, mean, samples };
const median = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);
const verdict = Object.entries(FLOORS).every(([key, floor]) => median[key as keyof typeof FLOORS] >= floor) ? 'pass' : 'fail';
return { judge: judge.name, bundle, verdict, median, mean, samples };
})));
const out = path.join(ROOT, 'docs/evals/judge-controls/results.json');
fs.mkdirSync(path.dirname(out), { recursive: true });
fs.writeFileSync(out, JSON.stringify({ recorded_at: new Date().toISOString(), results }, null, 2) + '\n');
for (const r of results) console.log(`${r.judge} | ${r.bundle} | ${r.verdict} | ${JSON.stringify(r.mean)}`);
for (const r of results) console.log(`${r.judge} | ${r.bundle} | ${r.verdict} | median ${JSON.stringify(r.median)} mean ${JSON.stringify(r.mean)}`);