feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

scripts/harness-version.json

@@ -1,5 +1,5 @@
 {
-  "version": 5,
+  "version": 6,
   "patterns": [
     "test/helpers/agent-sdk-runner.ts",
     "test/helpers/codex-session-runner.ts",
@@ -10,44 +10,44 @@
     "test/helpers/llm-judge.ts"
   ],
   "files": {
-    "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5},
-    "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5},
-    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
-    "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
-    "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
-    "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
-    "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
-    "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
-    "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
-    "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},
-    "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":5},
-    "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":5},
-    "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":5},
-    "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":5},
-    "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":5},
-    "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":5},
-    "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":5},
-    "test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":5},
-    "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":5},
-    "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":5},
-    "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":5},
-    "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":5},
-    "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":5},
-    "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":5},
-    "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":5},
-    "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":5},
-    "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":5},
-    "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":5},
-    "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":5},
-    "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":5},
-    "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":5},
-    "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":5},
-    "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":5},
-    "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":5},
-    "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":5},
-    "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":5},
-    "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":5},
-    "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":5},
-    "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":5}
+    "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6},
+    "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6},
+    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6},
+    "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
+    "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
+    "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6},
+    "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6},
+    "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
+    "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
+    "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
+    "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6},
+    "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6},
+    "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
+    "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6},
+    "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
+    "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6},
+    "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
+    "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
+    "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
+    "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
+    "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
+    "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6},
+    "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":6},
+    "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
+    "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
+    "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
+    "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6},
+    "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
+    "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
+    "test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":6},
+    "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
+    "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
+    "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
+    "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
+    "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6},
+    "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6},
+    "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6},
+    "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
+    "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
   }
 }

scripts/judge-calibration-configs.ts

@@ -13,7 +13,7 @@ import {
   judgePanelMedian, judgePanelMajority,
 } from '../test/helpers/llm-judge';
 import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
+import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
 import type { CalibrationConfig, Verdict } from './judge-calibration';
 
 type Sample = Record<string, unknown>;
@@ -40,7 +40,7 @@ const scoreJudge = (floors: Record<string, number>) => ({
 
 const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
   id, priority, cases,
-  model: extra.model ?? CLAUDE_FRONTIER_EVAL_MODEL,
+  model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL,
   build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability },
     { files: [], text: inputs.bundle! }),
   oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
@@ -63,19 +63,19 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
     estimatedOutputTokens: 300,
   },
   {
-    id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL,
     build: inputs => buildQaWorkflowJudgePrompt(inputs.section!),
     oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
     ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
   },
   {
-    id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL,
     build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!),
     oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
     ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
   },
   {
-    id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL,
     build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!),
     oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
     validate: value => requireFields(value, ['confidence'], ['would_browse']),
@@ -84,7 +84,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
     estimatedOutputTokens: 1200,
   },
   {
-    id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_JUDGE_EVAL_MODEL,
     build: inputs => buildCrossSkillConsistencyJudgePrompt(inputs.collected!),
     oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
     validate: value => requireFields(value, ['score'], ['consistent']),
@@ -93,7 +93,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
     estimatedOutputTokens: 1200,
   },
   {
-    id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_FRONTIER_EVAL_MODEL,
+    id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_JUDGE_EVAL_MODEL,
     build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
     oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
     validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),

scripts/models.ts

@@ -53,10 +53,10 @@ export type Model = (typeof ALL_MODEL_NAMES)[number];
  *      with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`,
  *      `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should
  *      not read nudges written for an older one.
- *    - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`,
- *      because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL
- *      (lib/eval-model.ts) and moving it would shift every eval baseline.
- *      The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes.
+ *    - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`
+ *      while a Fable 5 point release is one of gstack's eval models (the
+ *      judge, CLAUDE_JUDGE_EVAL_MODEL in lib/eval-model.ts). The exception
+ *      ends when CLAUDE_JUDGE_EVAL_MODEL leaves Fable 5.
  *    - Any other `claude-*` (including `claude-opus-5-5`) → `claude`
  *    - `gemini-*` (2.5-pro, flash, etc.) → `gemini`
  * 3. Unknown input → returns null (caller decides: error, or fall back).

scripts/resolvers/utility.ts

@@ -1,6 +1,6 @@
 import type { TemplateContext } from './types';
 import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG } from './constants';
-import { CLAUDE_FRONTIER_EVAL_MODEL } from '../../lib/eval-model';
+import { CLAUDE_CAPTURE_EVAL_MODEL } from '../../lib/eval-model';
 
 /**
  * {{CODEX_WEB_SEARCH_FLAG}} — the non-deprecated codex web-search flag
@@ -25,7 +25,7 @@ export function generateCodexReviewModelConfigFlag(_ctx: TemplateContext): strin
 }
 
 export function generateClaudeModelFlag(_ctx: TemplateContext): string {
-  return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_FRONTIER_EVAL_MODEL}}"`;
+  return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_CAPTURE_EVAL_MODEL}}"`;
 }
 
 export function generateSlugEval(ctx: TemplateContext): string {