diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31499643f..a00e4474b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,8 +165,10 @@ never touches a symlinked or non-gstack directory. ## Testing & evals Codex evals and the GPT benchmark adapter default to `gpt-6-astra`: -explicit model > `GSTACK_CODEX_MODEL` > default. Claude capture and judge -defaults are `claude-fable-5-1`, resolved through `lib/eval-model.ts`: +explicit model > `GSTACK_CODEX_MODEL` > default. The Claude model under test +(capture) defaults to `claude-opus-5-5` and the judge to `claude-fable-5-1`, +resolved through `lib/eval-model.ts`. The judge stays put when the model under +test moves, because its calibration fixtures are keyed to it: - Claude session, PTY, and Agent SDK eval runners and the Claude benchmark adapter: explicit model > `EVALS_MODEL` > `GSTACK_EVAL_MODEL_CAPTURE` > `GSTACK_EVAL_MODEL` > default. - Shared judge calls (including benchmark quality scoring): explicit model > `GSTACK_EVAL_MODEL_JUDGE` > `GSTACK_EVAL_MODEL` > default. `EVALS_MODEL` applies to capture runners, not judges. diff --git a/TODOS.md b/TODOS.md index 4afe20b8d..f76d0d5a4 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1651,7 +1651,8 @@ separately, so the change alters a paid verdict and needs its own paid run. `skill-e2e-design`, `skill-e2e-office-hours-phase4`, `skill-e2e-plan-prosons` and `skill-e2e-plan` keep `claude-opus-4-7` because six cases failed on the default model in one run (plan-design-review-plan-mode timeout, office-hours-phase4-fork format, plan-review-prosons-neutral-neg missing output, plan-ceo-review-selective and -plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). `skill-e2e-qa-bugs` returned +plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). plan-review-report, the one PR-gate +case in `skill-e2e-plan`, left the pin on 2026-10-18 and follows the capture default. `skill-e2e-qa-bugs` returned to `claude-opus-4-7` after `qa-b6-static` timed out on the default model in two of three runs (census 36597762183 and a targeted local rerun): each time the stream stopped mid-message, with no pending tool, right after the model found the disabled submit button, and emitted nothing until the 300 s case deadline. They measure an old model. diff --git a/lib/eval-model.ts b/lib/eval-model.ts index 75abe33cc..ba5917cad 100644 --- a/lib/eval-model.ts +++ b/lib/eval-model.ts @@ -12,26 +12,33 @@ * per-kind default — last resort * * Kinds and their defaults: - * capture — AskUserQuestion SDK capture runs: current frontier Claude model + * capture — the model under test in every Claude session, PTY and Agent SDK + * eval run: CLAUDE_CAPTURE_EVAL_MODEL * warmup — PTY warm-up ping (cheapest thing that answers): haiku * distill — free-text distillation (cheap, structured): haiku (pinned) - * judge — LLM-judge rubric calls: current frontier Claude model + * judge — LLM-judge rubric calls: CLAUDE_JUDGE_EVAL_MODEL */ -export const CLAUDE_FRONTIER_EVAL_MODEL = "claude-fable-5-1"; +/** The model under test: Anthropic's best code model (Garry, 2026-10-18). */ +export const CLAUDE_CAPTURE_EVAL_MODEL = "claude-opus-5-5"; +/** + * The judge is the ruler, not the model under test: its calibration fixtures + * (test/fixtures/judge-calibration) are keyed to it, so it moves only with a + * recalibration, never with the model under test. + */ +export const CLAUDE_JUDGE_EVAL_MODEL = "claude-fable-5-1"; export const DEFAULT_JUDGE_MAX_TOKENS = 8192; // `as const satisfies` keeps EvalModelKind the literal union // 'capture' | 'warmup' | 'distill' — a `Record` annotation // would widen it to string and let any typo through the type gate. const DEFAULTS = { - // Keep eval capture/judge on the current frontier Claude model by default. - // Tests needing a cheaper or historical ruler pass it explicitly or set + // Tests needing a cheaper or historical model pass it explicitly or set // GSTACK_EVAL_MODEL_CAPTURE / GSTACK_EVAL_MODEL_JUDGE / GSTACK_EVAL_MODEL. - capture: CLAUDE_FRONTIER_EVAL_MODEL, + capture: CLAUDE_CAPTURE_EVAL_MODEL, warmup: "claude-haiku-4-5", distill: "claude-haiku-4-5-20251001", - judge: CLAUDE_FRONTIER_EVAL_MODEL, + judge: CLAUDE_JUDGE_EVAL_MODEL, } as const satisfies Record; export type EvalModelKind = keyof typeof DEFAULTS; diff --git a/scripts/harness-version.json b/scripts/harness-version.json index a306a10bc..63ca100fb 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -1,5 +1,5 @@ { - "version": 5, + "version": 6, "patterns": [ "test/helpers/agent-sdk-runner.ts", "test/helpers/codex-session-runner.ts", @@ -10,44 +10,44 @@ "test/helpers/llm-judge.ts" ], "files": { - "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5}, - "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5}, - "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, - "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, - "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, - "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, - "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, - "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, - "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, - "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":5}, - "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":5}, - "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":5}, - "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":5}, - "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":5}, - "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":5}, - "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":5}, - "test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":5}, - "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":5}, - "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":5}, - "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":5}, - "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":5}, - "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":5}, - "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":5}, - "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":5}, - "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":5}, - "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":5}, - "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":5}, - "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":5}, - "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":5}, - "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":5}, - "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":5}, - "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":5}, - "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":5}, - "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":5}, - "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":5}, - "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":5}, - "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":5}, - "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":5} + "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6}, + "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6}, + "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6}, + "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, + "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, + "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, + "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6}, + "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, + "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, + "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, + "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, + "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6}, + "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, + "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6}, + "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, + "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6}, + "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, + "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, + "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, + "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6}, + "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6}, + "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6}, + "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":6}, + "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, + "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, + "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, + "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6}, + "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, + "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, + "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":6}, + "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, + "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, + "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, + "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, + "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6}, + "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6}, + "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6}, + "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, + "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} } } diff --git a/scripts/judge-calibration-configs.ts b/scripts/judge-calibration-configs.ts index fe5fe984b..426b3f8ba 100644 --- a/scripts/judge-calibration-configs.ts +++ b/scripts/judge-calibration-configs.ts @@ -13,7 +13,7 @@ import { judgePanelMedian, judgePanelMajority, } from '../test/helpers/llm-judge'; import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; +import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model'; import type { CalibrationConfig, Verdict } from './judge-calibration'; type Sample = Record; @@ -40,7 +40,7 @@ const scoreJudge = (floors: Record) => ({ const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record } = {}): CalibrationConfig => ({ id, priority, cases, - model: extra.model ?? CLAUDE_FRONTIER_EVAL_MODEL, + model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability }, { files: [], text: inputs.bundle! }), oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS }, @@ -63,19 +63,19 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [ estimatedOutputTokens: 300, }, { - id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_FRONTIER_EVAL_MODEL, + id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildQaWorkflowJudgePrompt(inputs.section!), oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA }, ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200, }, { - id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_FRONTIER_EVAL_MODEL, + id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!), oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA }, ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200, }, { - id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_FRONTIER_EVAL_MODEL, + id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!), oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA }, validate: value => requireFields(value, ['confidence'], ['would_browse']), @@ -84,7 +84,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [ estimatedOutputTokens: 1200, }, { - id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_FRONTIER_EVAL_MODEL, + id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildCrossSkillConsistencyJudgePrompt(inputs.collected!), oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA }, validate: value => requireFields(value, ['score'], ['consistent']), @@ -93,7 +93,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [ estimatedOutputTokens: 1200, }, { - id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_FRONTIER_EVAL_MODEL, + id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_JUDGE_EVAL_MODEL, build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!), oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA }, validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS), diff --git a/scripts/models.ts b/scripts/models.ts index 0c13ab371..fb80a0a1b 100644 --- a/scripts/models.ts +++ b/scripts/models.ts @@ -53,10 +53,10 @@ export type Model = (typeof ALL_MODEL_NAMES)[number]; * with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`, * `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should * not read nudges written for an older one. - * - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`, - * because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL - * (lib/eval-model.ts) and moving it would shift every eval baseline. - * The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes. + * - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5` + * while a Fable 5 point release is one of gstack's eval models (the + * judge, CLAUDE_JUDGE_EVAL_MODEL in lib/eval-model.ts). The exception + * ends when CLAUDE_JUDGE_EVAL_MODEL leaves Fable 5. * - Any other `claude-*` (including `claude-opus-5-5`) → `claude` * - `gemini-*` (2.5-pro, flash, etc.) → `gemini` * 3. Unknown input → returns null (caller decides: error, or fall back). diff --git a/scripts/resolvers/utility.ts b/scripts/resolvers/utility.ts index d5ffaf65b..2942bd451 100644 --- a/scripts/resolvers/utility.ts +++ b/scripts/resolvers/utility.ts @@ -1,6 +1,6 @@ import type { TemplateContext } from './types'; import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG } from './constants'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../../lib/eval-model'; +import { CLAUDE_CAPTURE_EVAL_MODEL } from '../../lib/eval-model'; /** * {{CODEX_WEB_SEARCH_FLAG}} — the non-deprecated codex web-search flag @@ -25,7 +25,7 @@ export function generateCodexReviewModelConfigFlag(_ctx: TemplateContext): strin } export function generateClaudeModelFlag(_ctx: TemplateContext): string { - return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_FRONTIER_EVAL_MODEL}}"`; + return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_CAPTURE_EVAL_MODEL}}"`; } export function generateSlugEval(ctx: TemplateContext): string { diff --git a/test/agent-sdk-runner.test.ts b/test/agent-sdk-runner.test.ts index 9284cdae4..78e2ea675 100644 --- a/test/agent-sdk-runner.test.ts +++ b/test/agent-sdk-runner.test.ts @@ -36,7 +36,7 @@ import { type OverlayFixture, } from '../test/fixtures/overlay-nudges'; import { firstAssistantMessageToolCount } from './helpers/overlay-measurement'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; +import { CLAUDE_CAPTURE_EVAL_MODEL } from '../lib/eval-model'; // --------------------------------------------------------------------------- // Stub SDK event builders @@ -47,7 +47,7 @@ function uuid(): string { return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`; } -function systemInit(model = CLAUDE_FRONTIER_EVAL_MODEL, version = '2.1.117'): SDKMessage { +function systemInit(model = CLAUDE_CAPTURE_EVAL_MODEL, version = '2.1.117'): SDKMessage { return { type: 'system', subtype: 'init', @@ -261,7 +261,7 @@ describe('runAgentSdkTest — happy path', () => { expect(result.turnsUsed).toBe(2); expect(result.costUsd).toBe(0.05); expect(result.sdkClaudeCodeVersion).toBe('2.1.117'); - expect(result.model).toBe(CLAUDE_FRONTIER_EVAL_MODEL); + expect(result.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL); expect(result.firstResponseMs).toBeGreaterThanOrEqual(0); }); @@ -845,7 +845,7 @@ describe('toSkillTestResult', () => { expect(s.output).toBe('hi'); expect(s.costEstimate.estimatedCost).toBe(0.02); expect(s.costEstimate.turnsUsed).toBe(1); - expect(s.model).toBe(CLAUDE_FRONTIER_EVAL_MODEL); + expect(s.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL); expect(s.firstResponseMs).toBeNumber(); expect(s.maxInterTurnMs).toBeNumber(); expect(s.transcript).toBeArray(); diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index f8e627960..071f761bc 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -14,7 +14,7 @@ import { test, expect } from 'bun:test'; import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from './helpers/benchmark-runner'; import { estimateCostUsd, PRICING } from './helpers/pricing'; import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; +import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model'; import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants'; test('estimateCostUsd returns 0 for unknown model (no crash)', () => { @@ -29,10 +29,10 @@ test('estimateCostUsd computes correctly for known Claude model', () => { expect(cost).toBeCloseTo(17.50, 2); }); -for (const model of [CLAUDE_FRONTIER_EVAL_MODEL, CODEX_FRONTIER_MODEL]) { +// Opus 5.5 costs $4/MTok input and $20/MTok output; Fable 5.1 (the judge) and Astra $10 and $50. +for (const [model, usd] of [[CLAUDE_CAPTURE_EVAL_MODEL, 0.008], [CLAUDE_JUDGE_EVAL_MODEL, 0.02], [CODEX_FRONTIER_MODEL, 0.02]] as const) { test(`estimateCostUsd prices the current ${model} default at its standard rate`, () => { - // Current default models both cost $10/MTok input and $50/MTok output. - expect(estimateCostUsd({ input: 1000, output: 200 }, model)).toBe(0.02); + expect(estimateCostUsd({ input: 1000, output: 200 }, model)).toBe(usd); }); } diff --git a/test/eval-model.test.ts b/test/eval-model.test.ts index 3432615e6..96e8a1dbf 100644 --- a/test/eval-model.test.ts +++ b/test/eval-model.test.ts @@ -1,6 +1,6 @@ /** OV11: the host-neutral eval-model resolver's contract. */ import { describe, test, expect } from "bun:test"; -import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from "../lib/eval-model"; +import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL, resolveEvalModel } from "../lib/eval-model"; describe("resolveEvalModel", () => { test("explicit argument wins over everything", () => { @@ -13,10 +13,10 @@ describe("resolveEvalModel", () => { expect(resolveEvalModel("distill", null, { GSTACK_EVAL_MODEL: "g" } as never)).toBe("g"); }); test("defaults per kind", () => { - expect(resolveEvalModel("capture", null, {} as never)).toBe(CLAUDE_FRONTIER_EVAL_MODEL); + expect(resolveEvalModel("capture", null, {} as never)).toBe(CLAUDE_CAPTURE_EVAL_MODEL); expect(resolveEvalModel("warmup", null, {} as never)).toBe("claude-haiku-4-5"); expect(resolveEvalModel("distill", null, {} as never)).toBe("claude-haiku-4-5-20251001"); - expect(resolveEvalModel("judge", null, {} as never)).toBe(CLAUDE_FRONTIER_EVAL_MODEL); + expect(resolveEvalModel("judge", null, {} as never)).toBe(CLAUDE_JUDGE_EVAL_MODEL); }); test("unknown kind throws instead of silently defaulting", () => { expect(() => resolveEvalModel("banana" as never, null, {} as never)).toThrow(); diff --git a/test/harness-version.test.ts b/test/harness-version.test.ts index bb54dd458..9f675cd62 100644 --- a/test/harness-version.test.ts +++ b/test/harness-version.test.ts @@ -8,7 +8,7 @@ */ import { describe, expect, test } from 'bun:test'; import * as path from 'node:path'; -import { decide, formatDrift, harnessDrift, hasDrift, readManifest, serializeManifest, workingHarnessFiles, +import { decide, formatDrift, harnessDrift, hasDrift, MANIFEST, readManifest, serializeManifest, workingHarnessFiles, type HarnessManifest } from '../scripts/bump-harness-version'; import { caseSeriesIdentitiesV2, globRegex, HARNESS_PATTERNS, HARNESS_VERSION, isCaseOwnedFile, skillDirs, stampTrialSeries, treeEntries, type TreeEntry } from '../scripts/eval-trial-series'; @@ -23,6 +23,7 @@ const ROOT = path.resolve(import.meta.dir, '..'); describe('HARNESS_VERSION guard', () => { test('every pinned shared-harness file matches scripts/harness-version.json', () => { + expect(MANIFEST).toBe('scripts/harness-version.json'); const manifest = readManifest(ROOT); expect(manifest.version).toBe(HARNESS_VERSION); expect(manifest.patterns).toEqual([...HARNESS_PATTERNS]); diff --git a/test/helpers/llm-judge.ts b/test/helpers/llm-judge.ts index 54e3306b4..d0bc6bbbe 100644 --- a/test/helpers/llm-judge.ts +++ b/test/helpers/llm-judge.ts @@ -13,7 +13,7 @@ import Anthropic from '@anthropic-ai/sdk'; import type { JSONOutputFormat } from '@anthropic-ai/sdk/resources/messages'; import { setTimeout as delay } from 'node:timers/promises'; -import { CLAUDE_FRONTIER_EVAL_MODEL, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model'; +import { CLAUDE_JUDGE_EVAL_MODEL, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model'; export { DEFAULT_JUDGE_MAX_TOKENS } from '../../lib/eval-model'; export interface JudgeScore { @@ -89,11 +89,12 @@ export interface RecommendationScore { /** * Call an Anthropic model with a prompt, extract JSON response. * Jittered exponential backoff over three 429 retries. Model resolves via - * lib/eval-model's `judge` kind (frontier Claude default); pass a model id + * lib/eval-model's `judge` kind (CLAUDE_JUDGE_EVAL_MODEL); pass a model id * (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like * judgeRecommendation. */ -// Default judge model: the current frontier Claude eval model. Override per run +// Default judge model: CLAUDE_JUDGE_EVAL_MODEL, kept apart from the model under +// test so calibration holds when the model under test moves. Override per run // with GSTACK_EVAL_MODEL_JUDGE; Haiku remains the right default for // classifier-grade duties (pty hung/working, warmup, distill — see // lib/eval-model.ts). @@ -125,7 +126,7 @@ export async function callJudge( const signal = opts?.signal; signal?.throwIfAborted(); // Routed through the documented single resolution point: explicit arg > - // GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > frontier default. The old + // GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > judge default. The old // inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global // GSTACK_EVAL_MODEL override that every other eval call site honors. // opts support bounded judgments; cancellation covers both requests and @@ -746,7 +747,7 @@ export interface ArmJudgeScore { * point of a research instrument; a per-run judge swap silently moves the * ruler. */ -export const ARM_JUDGE_MODEL = CLAUDE_FRONTIER_EVAL_MODEL; +export const ARM_JUDGE_MODEL = CLAUDE_JUDGE_EVAL_MODEL; /** * Build the over-engineering rubric prompt. Exported (pure) so the free diff --git a/test/model-catalog-isolation.test.ts b/test/model-catalog-isolation.test.ts index 1a649e8bc..83c6ed112 100644 --- a/test/model-catalog-isolation.test.ts +++ b/test/model-catalog-isolation.test.ts @@ -25,7 +25,7 @@ test('replacing the shipped catalog changes role choices, not eval rulers, overl fs.writeFileSync(entry, ` import { createHash } from 'node:crypto'; import { resolvePlanReviewModel, resolveImplementationModels } from './lib/model-policy'; -import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from './lib/eval-model'; +import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL, resolveEvalModel } from './lib/eval-model'; import { CODEX_FRONTIER_MODEL } from './scripts/resolvers/constants'; import { resolveCodexRuntimeModel, resolveCodexGenerationModel } from './scripts/resolve-codex-generation-model'; import { ALL_MODEL_NAMES, resolveClaudeOverlay } from './scripts/models'; @@ -37,7 +37,7 @@ console.log(JSON.stringify({ roles: ['anthropic', 'openai'].map(provider => resolvePlanReviewModel({ ...scope, provider }).requestedModel) .concat(resolveImplementationModels(scope).map(selection => selection.requestedModel)), independent: { - claudeRuler: CLAUDE_FRONTIER_EVAL_MODEL, openaiRuler: CODEX_FRONTIER_MODEL, + claudeRuler: CLAUDE_CAPTURE_EVAL_MODEL, claudeJudge: CLAUDE_JUDGE_EVAL_MODEL, openaiRuler: CODEX_FRONTIER_MODEL, capture: resolveEvalModel('capture'), judge: resolveEvalModel('judge'), codexExec: resolveCodexRuntimeModel({ kind: 'exec' }).model, codexReview: resolveCodexRuntimeModel({ kind: 'review' }).model, @@ -63,7 +63,7 @@ console.log(JSON.stringify({ const before = inspect(); expect(before.roles).toEqual(['claude-fable-5-1', 'gpt-6-astra', 'claude-opus-5-5', 'gpt-6.1-sol']); expect(before.independent).toMatchObject({ - claudeRuler: 'claude-fable-5-1', openaiRuler: 'gpt-6-astra', capture: 'claude-fable-5-1', + claudeRuler: 'claude-opus-5-5', claudeJudge: 'claude-fable-5-1', openaiRuler: 'gpt-6-astra', capture: 'claude-opus-5-5', judge: 'claude-fable-5-1', codexExec: 'gpt-6-astra', codexReview: 'gpt-6-astra', codexOverlay: 'gpt-6-astra', }); expect(before.independent.claudeArgs.some((arg: string) => arg.startsWith('--model'))).toBe(false); diff --git a/test/model-overlays.test.ts b/test/model-overlays.test.ts index 42307ad90..ce7fc4b23 100644 --- a/test/model-overlays.test.ts +++ b/test/model-overlays.test.ts @@ -10,7 +10,7 @@ import type { TemplateContext } from '../scripts/resolvers/types'; import { HOST_PATHS } from '../scripts/resolvers/types'; import { generateModelOverlay } from '../scripts/resolvers/model-overlay'; import { resolveModel, type Model } from '../scripts/models'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; +import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model'; import { readOverlay } from '../scripts/resolvers/model-overlay'; import { generateCompletenessSection } from '../scripts/resolvers/preamble/generate-completeness-section'; import { generateSetupCommand } from '../scripts/resolvers/utility'; @@ -407,9 +407,9 @@ describe('model-routing-claude', () => { expect(resolveModel('sonnet-5')).toBe('sonnet-5'); }); - test('Fable prefix exception holds only while the frontier eval model is a Fable 5 point release', () => { + test('Fable prefix exception holds only while the judge eval model is a Fable 5 point release', () => { if (resolveModel('claude-fable-5-99') === 'fable-5') { - expect(CLAUDE_FRONTIER_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true); + expect(CLAUDE_JUDGE_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true); } }); }); diff --git a/test/model-policy.test.ts b/test/model-policy.test.ts index edf327c74..b84eb31b9 100644 --- a/test/model-policy.test.ts +++ b/test/model-policy.test.ts @@ -20,7 +20,7 @@ import { validatePolicyValue, type ModelSelection, } from '../lib/model-policy'; -import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; +import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model'; import { CODEX_RUNTIME_MODEL_PATTERN } from '../scripts/resolve-codex-generation-model'; import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants'; @@ -99,7 +99,8 @@ describe('model catalog', () => { }); test('eval rulers and the no-role Codex pattern stay independent of the catalog', () => { - expect(CLAUDE_FRONTIER_EVAL_MODEL).toBe('claude-fable-5-1'); + expect(CLAUDE_CAPTURE_EVAL_MODEL).toBe('claude-opus-5-5'); + expect(CLAUDE_JUDGE_EVAL_MODEL).toBe('claude-fable-5-1'); expect(CODEX_FRONTIER_MODEL).toBe('gpt-6-astra'); for (const file of ['lib/eval-model.ts', 'scripts/resolvers/constants.ts']) { expect(fs.readFileSync(path.join(ROOT, file), 'utf-8')).not.toMatch(/from\s+['"][^'"]*model-(catalog|policy)/); diff --git a/test/plan-review-report-recording.test.ts b/test/plan-review-report-recording.test.ts index bbcf51841..77a31010a 100644 --- a/test/plan-review-report-recording.test.ts +++ b/test/plan-review-report-recording.test.ts @@ -8,6 +8,7 @@ import { isPreTurnInfraFailure, recordE2E } from './helpers/e2e-helpers'; import { EvalCollector, isFinalizedEvalResultFile, listEvalJsonFiles, type EvalTestEntry } from './helpers/eval-store'; import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt'; import { isPaidTestFile } from './helpers/paid-test-set'; +import { resolveEvalModel } from '../lib/eval-model'; const ROOT = path.resolve(import.meta.dir, '..'); const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-plan.test.ts'), 'utf8'); @@ -45,7 +46,7 @@ async function exercise(scenarios: Scenario[], opts: { deadline?: boolean; sibli for (const declaration of source.matchAll(/^import[\s\S]*?;\n/gm)) executable = executable.replace(declaration[0], ''); const js = new Bun.Transpiler({ loader: 'ts', target: 'bun' }).transformSync(executable); const args: Record = { - expect, ROOT, runId: 'public-report-free', evalsEnabled: true, CAPTURE_LONG_MS, + expect, ROOT, runId: 'public-report-free', evalsEnabled: true, CAPTURE_LONG_MS, resolveEvalModel, OFFICE_HOURS_BUN_GRACE_MS, EvalCollector: BoundCollector, recordE2E, createEvalCollector: (tier: string) => new BoundCollector(tier), finalizeEvalCollector: async (collector: EvalCollector | null) => { @@ -208,7 +209,7 @@ test('canonical reports preserve both successful exits, terminal forms, metadata expect(actual.helperOptions).toHaveLength(2); for (const options of actual.helperOptions) expect(options.budgetMs).toBe(CAPTURE_LONG_MS); for (const call of actual.calls) { - expect(call).toMatchObject({ testName: 'plan-review-report', model: 'claude-opus-4-7', maxTurns: 20, + expect(call).toMatchObject({ testName: 'plan-review-report', model: 'claude-opus-5-5', maxTurns: 20, timeout: CAPTURE_LONG_MS, runId: 'public-report-free' }); expect(call.signal).toBeInstanceOf(AbortSignal); expect(call.prompt).toContain('Read plan-eng-review/SKILL.md and plan-eng-review/sections/review-sections.md'); @@ -222,7 +223,7 @@ test('canonical reports preserve both successful exits, terminal forms, metadata for (const outcome of actual.outcomes) { expect(outcome.entries).toHaveLength(1); expect(outcome.entries[0]).toMatchObject({ name: '/plan-review-report', suite: 'Plan Review Report E2E', tier: 'e2e', passed: true, - duration_ms: 25, cost_usd: 0.12, model: 'claude-opus-4-7', first_response_ms: 3, max_inter_turn_ms: 7 }); + duration_ms: 25, cost_usd: 0.12, model: 'claude-opus-5-5', first_response_ms: 3, max_inter_turn_ms: 7 }); expect(outcome.entries[0].output).toHaveLength(2000); expect(outcome.entries[0].transcript).toEqual(outcome.result.transcript); } diff --git a/test/provider-model-defaults.test.ts b/test/provider-model-defaults.test.ts index 07ff031e1..a986a2d3a 100644 --- a/test/provider-model-defaults.test.ts +++ b/test/provider-model-defaults.test.ts @@ -332,9 +332,9 @@ describe.skipIf(process.platform === 'win32')('provider model selection', () => expect(await selected(adapter, 'custom-codex')).toBe('custom-codex'); }); - test('Claude defaults to Fable and preserves the full override chain', async () => { + test('Claude defaults to Opus 5.5 and preserves the full override chain', async () => { const adapter = new ClaudeAdapter(); - expect(await selected(adapter)).toBe('claude-fable-5-1'); + expect(await selected(adapter)).toBe('claude-opus-5-5'); process.env.GSTACK_EVAL_MODEL = 'global-model'; expect(await selected(adapter)).toBe('global-model'); process.env.GSTACK_EVAL_MODEL_CAPTURE = 'capture-model'; diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index fc60c1e73..92b79175c 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { EvalCollector } from './helpers/eval-store'; import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt'; import { @@ -382,7 +383,7 @@ describeIfSelected('Plan Review Report E2E', ['plan-review-report'], () => { try { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-report', suite: 'Plan Review Report E2E', - model: 'claude-opus-4-7', budgetMs: CAPTURE_LONG_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_LONG_MS, run: async signal => { planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-report-')); const run = (cmd: string, args: string[]) => { @@ -445,7 +446,7 @@ plan.md is the plan file for this review session; save your review there.`, timeout: CAPTURE_LONG_MS, testName: 'plan-review-report', runId, signal, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); }, validate: result => { diff --git a/test/third-party-actions-recording.test.ts b/test/third-party-actions-recording.test.ts index 5155363a7..9256303af 100644 --- a/test/third-party-actions-recording.test.ts +++ b/test/third-party-actions-recording.test.ts @@ -18,7 +18,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; const root = ${JSON.stringify(ROOT)}; const factsPath = ${JSON.stringify(facts)}; -const { CLAUDE_FRONTIER_EVAL_MODEL } = await import(path.join(root, 'lib/eval-model.ts')); +const { CLAUDE_CAPTURE_EVAL_MODEL } = await import(path.join(root, 'lib/eval-model.ts')); const readFile = fs.readFileSync.bind(fs); const recoveryResponses = JSON.parse(readFile(path.join(root, 'test/fixtures/third-party-actions-recovery-public.json'), 'utf8')).responses; const writeFile = fs.writeFileSync.bind(fs); @@ -128,7 +128,7 @@ test('actual paid assertion boundaries and all failure stages retain one accurat expect(observed.created).toEqual([]); if (kind === 'runner' || kind === 'fixture') { expect(observed.error).toBe(kind === 'runner' ? runnerError : fixtureError); - expect(observed.entry).toMatchObject({ exit_reason: 'harness_error', cost_usd: 0, model: CLAUDE_FRONTIER_EVAL_MODEL }); + expect(observed.entry).toMatchObject({ exit_reason: 'harness_error', cost_usd: 0, model: CLAUDE_CAPTURE_EVAL_MODEL }); expect(observed.entry.error).toContain('cost and usage unavailable'); expect(observed.entry.transcript).toBeUndefined(); expect(observed.runnerCalls).toBe(kind === 'runner' ? 1 : 0); @@ -168,7 +168,7 @@ test('actual paid assertion boundaries and all failure stages retain one accurat // A runner/setup exception has no returned model. Mirror the same capture // resolution that the real session runner would have used, including overrides. for (const [overrides, expected] of [ - [{}, CLAUDE_FRONTIER_EVAL_MODEL], + [{}, CLAUDE_CAPTURE_EVAL_MODEL], [{ GSTACK_EVAL_MODEL: 'global-model' }, 'global-model'], [{ GSTACK_EVAL_MODEL: 'global-model', GSTACK_EVAL_MODEL_CAPTURE: 'capture-model' }, 'capture-model'], [{ GSTACK_EVAL_MODEL: 'global-model', GSTACK_EVAL_MODEL_CAPTURE: 'capture-model', EVALS_MODEL: 'evals-model' }, 'evals-model'],