feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-11-04 15:57:33 +01:00 · 2026-10-18 19:09:54 +00:00
1 parent e9dfcf64f0
commit 4ce7f32d31
@@ -165,8 +165,10 @@ never touches a symlinked or non-gstack directory.
 ## Testing & evals

 Codex evals and the GPT benchmark adapter default to `gpt-6-astra`:
-explicit model > `GSTACK_CODEX_MODEL` > default. Claude capture and judge
-defaults are `claude-fable-5-1`, resolved through `lib/eval-model.ts`:
+explicit model > `GSTACK_CODEX_MODEL` > default. The Claude model under test
+(capture) defaults to `claude-opus-5-5` and the judge to `claude-fable-5-1`,
+resolved through `lib/eval-model.ts`. The judge stays put when the model under
+test moves, because its calibration fixtures are keyed to it:

 - Claude session, PTY, and Agent SDK eval runners and the Claude benchmark adapter: explicit model > `EVALS_MODEL` > `GSTACK_EVAL_MODEL_CAPTURE` > `GSTACK_EVAL_MODEL` > default.
 - Shared judge calls (including benchmark quality scoring): explicit model > `GSTACK_EVAL_MODEL_JUDGE` > `GSTACK_EVAL_MODEL` > default. `EVALS_MODEL` applies to capture runners, not judges.
@@ -1651,7 +1651,8 @@ separately, so the change alters a paid verdict and needs its own paid run.
 `skill-e2e-design`, `skill-e2e-office-hours-phase4`, `skill-e2e-plan-prosons` and `skill-e2e-plan` keep
 `claude-opus-4-7` because six cases failed on the default model in one run (plan-design-review-plan-mode timeout,
 office-hours-phase4-fork format, plan-review-prosons-neutral-neg missing output, plan-ceo-review-selective and
-plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). `skill-e2e-qa-bugs` returned
+plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). plan-review-report, the one PR-gate
+case in `skill-e2e-plan`, left the pin on 2026-10-18 and follows the capture default. `skill-e2e-qa-bugs` returned
 to `claude-opus-4-7` after `qa-b6-static` timed out on the default model in two of three runs (census 36597762183
 and a targeted local rerun): each time the stream stopped mid-message, with no pending tool, right after the model
 found the disabled submit button, and emitted nothing until the 300 s case deadline. They measure an old model.
@@ -12,26 +12,33 @@
 *   per-kind default                                         — last resort
 *
 * Kinds and their defaults:
- *   capture — AskUserQuestion SDK capture runs: current frontier Claude model
+ *   capture — the model under test in every Claude session, PTY and Agent SDK
+ *             eval run: CLAUDE_CAPTURE_EVAL_MODEL
 *   warmup  — PTY warm-up ping (cheapest thing that answers): haiku
 *   distill — free-text distillation (cheap, structured): haiku (pinned)
- *   judge   — LLM-judge rubric calls: current frontier Claude model
+ *   judge   — LLM-judge rubric calls: CLAUDE_JUDGE_EVAL_MODEL
 */

-export const CLAUDE_FRONTIER_EVAL_MODEL = "claude-fable-5-1";
+/** The model under test: Anthropic's best code model (Garry, 2026-10-18). */
+export const CLAUDE_CAPTURE_EVAL_MODEL = "claude-opus-5-5";
+/**
+ * The judge is the ruler, not the model under test: its calibration fixtures
+ * (test/fixtures/judge-calibration) are keyed to it, so it moves only with a
+ * recalibration, never with the model under test.
+ */
+export const CLAUDE_JUDGE_EVAL_MODEL = "claude-fable-5-1";
 export const DEFAULT_JUDGE_MAX_TOKENS = 8192;

 // `as const satisfies` keeps EvalModelKind the literal union
 // 'capture' | 'warmup' | 'distill' — a `Record<string, string>` annotation
 // would widen it to string and let any typo through the type gate.
 const DEFAULTS = {
-  // Keep eval capture/judge on the current frontier Claude model by default.
-  // Tests needing a cheaper or historical ruler pass it explicitly or set
+  // Tests needing a cheaper or historical model pass it explicitly or set
  // GSTACK_EVAL_MODEL_CAPTURE / GSTACK_EVAL_MODEL_JUDGE / GSTACK_EVAL_MODEL.
-  capture: CLAUDE_FRONTIER_EVAL_MODEL,
+  capture: CLAUDE_CAPTURE_EVAL_MODEL,
  warmup: "claude-haiku-4-5",
  distill: "claude-haiku-4-5-20251001",
-  judge: CLAUDE_FRONTIER_EVAL_MODEL,
+  judge: CLAUDE_JUDGE_EVAL_MODEL,
 } as const satisfies Record<string, string>;

 export type EvalModelKind = keyof typeof DEFAULTS;
@@ -1,5 +1,5 @@
 {
-  "version": 5,
+  "version": 6,
  "patterns": [
    "test/helpers/agent-sdk-runner.ts",
    "test/helpers/codex-session-runner.ts",
@@ -10,44 +10,44 @@
    "test/helpers/llm-judge.ts"
  ],
  "files": {
-    "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5},
-    "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5},
-    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
-    "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
-    "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
-    "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
-    "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
-    "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
-    "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
-    "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},
-    "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":5},
-    "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":5},
-    "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":5},
-    "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":5},
-    "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":5},
-    "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":5},
-    "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":5},
-    "test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":5},
-    "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":5},
-    "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":5},
-    "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":5},
-    "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":5},
-    "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":5},
-    "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":5},
-    "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":5},
-    "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":5},
-    "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":5},
-    "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":5},
-    "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":5},
-    "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":5},
-    "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":5},
-    "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":5},
-    "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":5},
-    "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":5},
-    "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":5},
-    "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":5},
-    "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":5},
-    "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":5},
-    "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":5}
+    "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6},
+    "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6},
+    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6},
+    "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
+    "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
+    "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6},
+    "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6},
+    "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
+    "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
+    "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
+    "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6},
+    "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6},
+    "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
+    "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6},
+    "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
+    "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6},
+    "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
+    "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
+    "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
+    "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
+    "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
+    "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6},
+    "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":6},
+    "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
+    "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
+    "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
+    "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6},
+    "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
+    "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
+    "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":6},
+    "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
+    "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
+    "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
+    "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
+    "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6},
+    "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6},
+    "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6},
+    "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
+    "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
  }
 }
@@ -13,7 +13,7 @@ import {
  judgePanelMedian, judgePanelMajority,
 } from '../test/helpers/llm-judge';
 import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
 import type { CalibrationConfig, Verdict } from './judge-calibration';

 type Sample = Record<string, unknown>;
@@ -40,7 +40,7 @@ const scoreJudge = (floors: Record<string, number>) => ({

 const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
  id, priority, cases,
-  model: extra.model ?? CLAUDE_FRONTIER_EVAL_MODEL,
+  model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL,
  build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability },
    { files: [], text: inputs.bundle! }),
  oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
@@ -63,19 +63,19 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
    estimatedOutputTokens: 300,
  },
  {
-    id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL,
    build: inputs => buildQaWorkflowJudgePrompt(inputs.section!),
    oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
    ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
  },
  {
-    id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL,
    build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!),
    oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
    ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
  },
  {
-    id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL,
    build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!),
    oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
    validate: value => requireFields(value, ['confidence'], ['would_browse']),
@@ -84,7 +84,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
    estimatedOutputTokens: 1200,
  },
  {
-    id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_JUDGE_EVAL_MODEL,
    build: inputs => buildCrossSkillConsistencyJudgePrompt(inputs.collected!),
    oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
    validate: value => requireFields(value, ['score'], ['consistent']),
@@ -93,7 +93,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
    estimatedOutputTokens: 1200,
  },
  {
-    id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_JUDGE_EVAL_MODEL,
    build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
    oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
    validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
@@ -53,10 +53,10 @@ export type Model = (typeof ALL_MODEL_NAMES)[number];
 *      with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`,
 *      `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should
 *      not read nudges written for an older one.
- *    - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`,
- *      because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL
- *      (lib/eval-model.ts) and moving it would shift every eval baseline.
- *      The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes.
+ *    - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`
+ *      while a Fable 5 point release is one of gstack's eval models (the
+ *      judge, CLAUDE_JUDGE_EVAL_MODEL in lib/eval-model.ts). The exception
+ *      ends when CLAUDE_JUDGE_EVAL_MODEL leaves Fable 5.
 *    - Any other `claude-*` (including `claude-opus-5-5`) → `claude`
 *    - `gemini-*` (2.5-pro, flash, etc.) → `gemini`
 * 3. Unknown input → returns null (caller decides: error, or fall back).
@@ -1,6 +1,6 @@
 import type { TemplateContext } from './types';
 import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG } from './constants';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../../lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL } from '../../lib/eval-model';

 /**
 * {{CODEX_WEB_SEARCH_FLAG}} — the non-deprecated codex web-search flag
@@ -25,7 +25,7 @@ export function generateCodexReviewModelConfigFlag(_ctx: TemplateContext): strin
 }

 export function generateClaudeModelFlag(_ctx: TemplateContext): string {
-  return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_FRONTIER_EVAL_MODEL}}"`;
+  return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_CAPTURE_EVAL_MODEL}}"`;
 }

 export function generateSlugEval(ctx: TemplateContext): string {
@@ -36,7 +36,7 @@ import {
  type OverlayFixture,
 } from '../test/fixtures/overlay-nudges';
 import { firstAssistantMessageToolCount } from './helpers/overlay-measurement';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL } from '../lib/eval-model';

 // ---------------------------------------------------------------------------
 // Stub SDK event builders
@@ -47,7 +47,7 @@ function uuid(): string {
  return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
 }

-function systemInit(model = CLAUDE_FRONTIER_EVAL_MODEL, version = '2.1.117'): SDKMessage {
+function systemInit(model = CLAUDE_CAPTURE_EVAL_MODEL, version = '2.1.117'): SDKMessage {
  return {
    type: 'system',
    subtype: 'init',
@@ -261,7 +261,7 @@ describe('runAgentSdkTest — happy path', () => {
    expect(result.turnsUsed).toBe(2);
    expect(result.costUsd).toBe(0.05);
    expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
-    expect(result.model).toBe(CLAUDE_FRONTIER_EVAL_MODEL);
+    expect(result.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL);
    expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
  });

@@ -845,7 +845,7 @@ describe('toSkillTestResult', () => {
    expect(s.output).toBe('hi');
    expect(s.costEstimate.estimatedCost).toBe(0.02);
    expect(s.costEstimate.turnsUsed).toBe(1);
-    expect(s.model).toBe(CLAUDE_FRONTIER_EVAL_MODEL);
+    expect(s.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL);
    expect(s.firstResponseMs).toBeNumber();
    expect(s.maxInterTurnMs).toBeNumber();
    expect(s.transcript).toBeArray();
@@ -14,7 +14,7 @@ import { test, expect } from 'bun:test';
 import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from './helpers/benchmark-runner';
 import { estimateCostUsd, PRICING } from './helpers/pricing';
 import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
 import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';

 test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
@@ -29,10 +29,10 @@ test('estimateCostUsd computes correctly for known Claude model', () => {
  expect(cost).toBeCloseTo(17.50, 2);
 });

-for (const model of [CLAUDE_FRONTIER_EVAL_MODEL, CODEX_FRONTIER_MODEL]) {
+// Opus 5.5 costs $4/MTok input and $20/MTok output; Fable 5.1 (the judge) and Astra $10 and $50.
+for (const [model, usd] of [[CLAUDE_CAPTURE_EVAL_MODEL, 0.008], [CLAUDE_JUDGE_EVAL_MODEL, 0.02], [CODEX_FRONTIER_MODEL, 0.02]] as const) {
  test(`estimateCostUsd prices the current ${model} default at its standard rate`, () => {
-    // Current default models both cost $10/MTok input and $50/MTok output.
-    expect(estimateCostUsd({ input: 1000, output: 200 }, model)).toBe(0.02);
+    expect(estimateCostUsd({ input: 1000, output: 200 }, model)).toBe(usd);
  });
 }

@@ -1,6 +1,6 @@
 /** OV11: the host-neutral eval-model resolver's contract. */
 import { describe, test, expect } from "bun:test";
-import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from "../lib/eval-model";
+import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL, resolveEvalModel } from "../lib/eval-model";

 describe("resolveEvalModel", () => {
  test("explicit argument wins over everything", () => {
@@ -13,10 +13,10 @@ describe("resolveEvalModel", () => {
    expect(resolveEvalModel("distill", null, { GSTACK_EVAL_MODEL: "g" } as never)).toBe("g");
  });
  test("defaults per kind", () => {
-    expect(resolveEvalModel("capture", null, {} as never)).toBe(CLAUDE_FRONTIER_EVAL_MODEL);
+    expect(resolveEvalModel("capture", null, {} as never)).toBe(CLAUDE_CAPTURE_EVAL_MODEL);
    expect(resolveEvalModel("warmup", null, {} as never)).toBe("claude-haiku-4-5");
    expect(resolveEvalModel("distill", null, {} as never)).toBe("claude-haiku-4-5-20251001");
-    expect(resolveEvalModel("judge", null, {} as never)).toBe(CLAUDE_FRONTIER_EVAL_MODEL);
+    expect(resolveEvalModel("judge", null, {} as never)).toBe(CLAUDE_JUDGE_EVAL_MODEL);
  });
  test("unknown kind throws instead of silently defaulting", () => {
    expect(() => resolveEvalModel("banana" as never, null, {} as never)).toThrow();
@@ -8,7 +8,7 @@
 */
 import { describe, expect, test } from 'bun:test';
 import * as path from 'node:path';
-import { decide, formatDrift, harnessDrift, hasDrift, readManifest, serializeManifest, workingHarnessFiles,
+import { decide, formatDrift, harnessDrift, hasDrift, MANIFEST, readManifest, serializeManifest, workingHarnessFiles,
  type HarnessManifest } from '../scripts/bump-harness-version';
 import { caseSeriesIdentitiesV2, globRegex, HARNESS_PATTERNS, HARNESS_VERSION, isCaseOwnedFile, skillDirs, stampTrialSeries, treeEntries,
  type TreeEntry } from '../scripts/eval-trial-series';
@@ -23,6 +23,7 @@ const ROOT = path.resolve(import.meta.dir, '..');

 describe('HARNESS_VERSION guard', () => {
  test('every pinned shared-harness file matches scripts/harness-version.json', () => {
+    expect(MANIFEST).toBe('scripts/harness-version.json');
    const manifest = readManifest(ROOT);
    expect(manifest.version).toBe(HARNESS_VERSION);
    expect(manifest.patterns).toEqual([...HARNESS_PATTERNS]);
@@ -13,7 +13,7 @@ import Anthropic from '@anthropic-ai/sdk';
 import type { JSONOutputFormat } from '@anthropic-ai/sdk/resources/messages';
 import { setTimeout as delay } from 'node:timers/promises';

-import { CLAUDE_FRONTIER_EVAL_MODEL, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
+import { CLAUDE_JUDGE_EVAL_MODEL, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
 export { DEFAULT_JUDGE_MAX_TOKENS } from '../../lib/eval-model';

 export interface JudgeScore {
@@ -89,11 +89,12 @@ export interface RecommendationScore {
 /**
 * Call an Anthropic model with a prompt, extract JSON response.
 * Jittered exponential backoff over three 429 retries. Model resolves via
- * lib/eval-model's `judge` kind (frontier Claude default); pass a model id
+ * lib/eval-model's `judge` kind (CLAUDE_JUDGE_EVAL_MODEL); pass a model id
 * (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like
 * judgeRecommendation.
 */
-// Default judge model: the current frontier Claude eval model. Override per run
+// Default judge model: CLAUDE_JUDGE_EVAL_MODEL, kept apart from the model under
+// test so calibration holds when the model under test moves. Override per run
 // with GSTACK_EVAL_MODEL_JUDGE; Haiku remains the right default for
 // classifier-grade duties (pty hung/working, warmup, distill — see
 // lib/eval-model.ts).
@@ -125,7 +126,7 @@ export async function callJudge<T>(
  const signal = opts?.signal;
  signal?.throwIfAborted();
  // Routed through the documented single resolution point: explicit arg >
-  // GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > frontier default. The old
+  // GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > judge default. The old
  // inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global
  // GSTACK_EVAL_MODEL override that every other eval call site honors.
  // opts support bounded judgments; cancellation covers both requests and
@@ -746,7 +747,7 @@ export interface ArmJudgeScore {
 * point of a research instrument; a per-run judge swap silently moves the
 * ruler.
 */
-export const ARM_JUDGE_MODEL = CLAUDE_FRONTIER_EVAL_MODEL;
+export const ARM_JUDGE_MODEL = CLAUDE_JUDGE_EVAL_MODEL;

 /**
 * Build the over-engineering rubric prompt. Exported (pure) so the free
@@ -25,7 +25,7 @@ test('replacing the shipped catalog changes role choices, not eval rulers, overl
    fs.writeFileSync(entry, `
 import { createHash } from 'node:crypto';
 import { resolvePlanReviewModel, resolveImplementationModels } from './lib/model-policy';
-import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from './lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL, resolveEvalModel } from './lib/eval-model';
 import { CODEX_FRONTIER_MODEL } from './scripts/resolvers/constants';
 import { resolveCodexRuntimeModel, resolveCodexGenerationModel } from './scripts/resolve-codex-generation-model';
 import { ALL_MODEL_NAMES, resolveClaudeOverlay } from './scripts/models';
@@ -37,7 +37,7 @@ console.log(JSON.stringify({
  roles: ['anthropic', 'openai'].map(provider => resolvePlanReviewModel({ ...scope, provider }).requestedModel)
    .concat(resolveImplementationModels(scope).map(selection => selection.requestedModel)),
  independent: {
-    claudeRuler: CLAUDE_FRONTIER_EVAL_MODEL, openaiRuler: CODEX_FRONTIER_MODEL,
+    claudeRuler: CLAUDE_CAPTURE_EVAL_MODEL, claudeJudge: CLAUDE_JUDGE_EVAL_MODEL, openaiRuler: CODEX_FRONTIER_MODEL,
    capture: resolveEvalModel('capture'), judge: resolveEvalModel('judge'),
    codexExec: resolveCodexRuntimeModel({ kind: 'exec' }).model,
    codexReview: resolveCodexRuntimeModel({ kind: 'review' }).model,
@@ -63,7 +63,7 @@ console.log(JSON.stringify({
    const before = inspect();
    expect(before.roles).toEqual(['claude-fable-5-1', 'gpt-6-astra', 'claude-opus-5-5', 'gpt-6.1-sol']);
    expect(before.independent).toMatchObject({
-      claudeRuler: 'claude-fable-5-1', openaiRuler: 'gpt-6-astra', capture: 'claude-fable-5-1',
+      claudeRuler: 'claude-opus-5-5', claudeJudge: 'claude-fable-5-1', openaiRuler: 'gpt-6-astra', capture: 'claude-opus-5-5',
      judge: 'claude-fable-5-1', codexExec: 'gpt-6-astra', codexReview: 'gpt-6-astra', codexOverlay: 'gpt-6-astra',
    });
    expect(before.independent.claudeArgs.some((arg: string) => arg.startsWith('--model'))).toBe(false);
@@ -10,7 +10,7 @@ import type { TemplateContext } from '../scripts/resolvers/types';
 import { HOST_PATHS } from '../scripts/resolvers/types';
 import { generateModelOverlay } from '../scripts/resolvers/model-overlay';
 import { resolveModel, type Model } from '../scripts/models';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
 import { readOverlay } from '../scripts/resolvers/model-overlay';
 import { generateCompletenessSection } from '../scripts/resolvers/preamble/generate-completeness-section';
 import { generateSetupCommand } from '../scripts/resolvers/utility';
@@ -407,9 +407,9 @@ describe('model-routing-claude', () => {
    expect(resolveModel('sonnet-5')).toBe('sonnet-5');
  });

-  test('Fable prefix exception holds only while the frontier eval model is a Fable 5 point release', () => {
+  test('Fable prefix exception holds only while the judge eval model is a Fable 5 point release', () => {
    if (resolveModel('claude-fable-5-99') === 'fable-5') {
-      expect(CLAUDE_FRONTIER_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true);
+      expect(CLAUDE_JUDGE_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true);
    }
  });
 });
@@ -20,7 +20,7 @@ import {
  validatePolicyValue,
  type ModelSelection,
 } from '../lib/model-policy';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
 import { CODEX_RUNTIME_MODEL_PATTERN } from '../scripts/resolve-codex-generation-model';
 import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';

@@ -99,7 +99,8 @@ describe('model catalog', () => {
  });

  test('eval rulers and the no-role Codex pattern stay independent of the catalog', () => {
-    expect(CLAUDE_FRONTIER_EVAL_MODEL).toBe('claude-fable-5-1');
+    expect(CLAUDE_CAPTURE_EVAL_MODEL).toBe('claude-opus-5-5');
+    expect(CLAUDE_JUDGE_EVAL_MODEL).toBe('claude-fable-5-1');
    expect(CODEX_FRONTIER_MODEL).toBe('gpt-6-astra');
    for (const file of ['lib/eval-model.ts', 'scripts/resolvers/constants.ts']) {
      expect(fs.readFileSync(path.join(ROOT, file), 'utf-8')).not.toMatch(/from\s+['"][^'"]*model-(catalog|policy)/);
@@ -8,6 +8,7 @@ import { isPreTurnInfraFailure, recordE2E } from './helpers/e2e-helpers';
 import { EvalCollector, isFinalizedEvalResultFile, listEvalJsonFiles, type EvalTestEntry } from './helpers/eval-store';
 import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';
 import { isPaidTestFile } from './helpers/paid-test-set';
+import { resolveEvalModel } from '../lib/eval-model';

 const ROOT = path.resolve(import.meta.dir, '..');
 const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-plan.test.ts'), 'utf8');
@@ -45,7 +46,7 @@ async function exercise(scenarios: Scenario[], opts: { deadline?: boolean; sibli
  for (const declaration of source.matchAll(/^import[\s\S]*?;\n/gm)) executable = executable.replace(declaration[0], '');
  const js = new Bun.Transpiler({ loader: 'ts', target: 'bun' }).transformSync(executable);
  const args: Record<string, any> = {
-    expect, ROOT, runId: 'public-report-free', evalsEnabled: true, CAPTURE_LONG_MS,
+    expect, ROOT, runId: 'public-report-free', evalsEnabled: true, CAPTURE_LONG_MS, resolveEvalModel,
    OFFICE_HOURS_BUN_GRACE_MS, EvalCollector: BoundCollector, recordE2E,
    createEvalCollector: (tier: string) => new BoundCollector(tier),
    finalizeEvalCollector: async (collector: EvalCollector | null) => {
@@ -208,7 +209,7 @@ test('canonical reports preserve both successful exits, terminal forms, metadata
  expect(actual.helperOptions).toHaveLength(2);
  for (const options of actual.helperOptions) expect(options.budgetMs).toBe(CAPTURE_LONG_MS);
  for (const call of actual.calls) {
-    expect(call).toMatchObject({ testName: 'plan-review-report', model: 'claude-opus-4-7', maxTurns: 20,
+    expect(call).toMatchObject({ testName: 'plan-review-report', model: 'claude-opus-5-5', maxTurns: 20,
      timeout: CAPTURE_LONG_MS, runId: 'public-report-free' });
    expect(call.signal).toBeInstanceOf(AbortSignal);
    expect(call.prompt).toContain('Read plan-eng-review/SKILL.md and plan-eng-review/sections/review-sections.md');
@@ -222,7 +223,7 @@ test('canonical reports preserve both successful exits, terminal forms, metadata
  for (const outcome of actual.outcomes) {
    expect(outcome.entries).toHaveLength(1);
    expect(outcome.entries[0]).toMatchObject({ name: '/plan-review-report', suite: 'Plan Review Report E2E', tier: 'e2e', passed: true,
-      duration_ms: 25, cost_usd: 0.12, model: 'claude-opus-4-7', first_response_ms: 3, max_inter_turn_ms: 7 });
+      duration_ms: 25, cost_usd: 0.12, model: 'claude-opus-5-5', first_response_ms: 3, max_inter_turn_ms: 7 });
    expect(outcome.entries[0].output).toHaveLength(2000);
    expect(outcome.entries[0].transcript).toEqual(outcome.result.transcript);
  }
@@ -332,9 +332,9 @@ describe.skipIf(process.platform === 'win32')('provider model selection', () =>
    expect(await selected(adapter, 'custom-codex')).toBe('custom-codex');
  });

-  test('Claude defaults to Fable and preserves the full override chain', async () => {
+  test('Claude defaults to Opus 5.5 and preserves the full override chain', async () => {
    const adapter = new ClaudeAdapter();
-    expect(await selected(adapter)).toBe('claude-fable-5-1');
+    expect(await selected(adapter)).toBe('claude-opus-5-5');
    process.env.GSTACK_EVAL_MODEL = 'global-model';
    expect(await selected(adapter)).toBe('global-model');
    process.env.GSTACK_EVAL_MODEL_CAPTURE = 'capture-model';
@@ -1,6 +1,7 @@
 import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
 import { runSkillTest } from './helpers/session-runner';
+import { resolveEvalModel } from '../lib/eval-model';
 import { EvalCollector } from './helpers/eval-store';
 import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';
 import {
@@ -382,7 +383,7 @@ describeIfSelected('Plan Review Report E2E', ['plan-review-report'], () => {
    try {
      await runRecordedOfficeHoursAttempt({
        collector: evalCollector, name: '/plan-review-report', suite: 'Plan Review Report E2E',
-        model: 'claude-opus-4-7', budgetMs: CAPTURE_LONG_MS,
+        model: resolveEvalModel('capture'), budgetMs: CAPTURE_LONG_MS,
        run: async signal => {
          planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-report-'));
          const run = (cmd: string, args: string[]) => {
@@ -445,7 +446,7 @@ plan.md is the plan file for this review session; save your review there.`,
            timeout: CAPTURE_LONG_MS,
            testName: 'plan-review-report',
            runId, signal,
-            model: 'claude-opus-4-7',
+            model: resolveEvalModel('capture'),
          });
        },
        validate: result => {
@@ -18,7 +18,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 const root = ${JSON.stringify(ROOT)};
 const factsPath = ${JSON.stringify(facts)};
-const { CLAUDE_FRONTIER_EVAL_MODEL } = await import(path.join(root, 'lib/eval-model.ts'));
+const { CLAUDE_CAPTURE_EVAL_MODEL } = await import(path.join(root, 'lib/eval-model.ts'));
 const readFile = fs.readFileSync.bind(fs);
 const recoveryResponses = JSON.parse(readFile(path.join(root, 'test/fixtures/third-party-actions-recovery-public.json'), 'utf8')).responses;
 const writeFile = fs.writeFileSync.bind(fs);
@@ -128,7 +128,7 @@ test('actual paid assertion boundaries and all failure stages retain one accurat
      expect(observed.created).toEqual([]);
      if (kind === 'runner' || kind === 'fixture') {
        expect(observed.error).toBe(kind === 'runner' ? runnerError : fixtureError);
-        expect(observed.entry).toMatchObject({ exit_reason: 'harness_error', cost_usd: 0, model: CLAUDE_FRONTIER_EVAL_MODEL });
+        expect(observed.entry).toMatchObject({ exit_reason: 'harness_error', cost_usd: 0, model: CLAUDE_CAPTURE_EVAL_MODEL });
        expect(observed.entry.error).toContain('cost and usage unavailable');
        expect(observed.entry.transcript).toBeUndefined();
        expect(observed.runnerCalls).toBe(kind === 'runner' ? 1 : 0);
@@ -168,7 +168,7 @@ test('actual paid assertion boundaries and all failure stages retain one accurat
  // A runner/setup exception has no returned model. Mirror the same capture
  // resolution that the real session runner would have used, including overrides.
  for (const [overrides, expected] of [
-    [{}, CLAUDE_FRONTIER_EVAL_MODEL],
+    [{}, CLAUDE_CAPTURE_EVAL_MODEL],
    [{ GSTACK_EVAL_MODEL: 'global-model' }, 'global-model'],
    [{ GSTACK_EVAL_MODEL: 'global-model', GSTACK_EVAL_MODEL_CAPTURE: 'capture-model' }, 'capture-model'],
    [{ GSTACK_EVAL_MODEL: 'global-model', GSTACK_EVAL_MODEL_CAPTURE: 'capture-model', EVALS_MODEL: 'evals-model' }, 'evals-model'],