From 2d9d155dd5482ed75ff1df38c0032d9b4c95fe39 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:40:01 +0000 Subject: [PATCH] fix(models): allowlist routing for model-pinned Claude families (W1) resolveModel lowercases input and strips one trailing [...] context marker. Opus 4.7, Opus 4.8 and Sonnet 5 keep their pinned overlay only for the bare ID, the bare ID plus an 8-digit date, and the bare ID plus -latest; any other suffix (point releases such as claude-sonnet-5-5, claude-opus-4-7-1) falls to the generic claude overlay. claude-opus-5-5 routes to claude explicitly (no new overlay, no new ALL_MODEL_NAMES entry). Fable keeps its prefix match while CLAUDE_FRONTIER_EVAL_MODEL is claude-fable-5-1, guarded by a test. Tests: one test/model-overlays.test.ts case per shape in the plan, plus the lowercase / [1m] marker / family-name cases and the Fable guard. --- scripts/models.ts | 25 ++++++++++++++++----- test/model-overlays.test.ts | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/scripts/models.ts b/scripts/models.ts index cfa462032..f80a14cd1 100644 --- a/scripts/models.ts +++ b/scripts/models.ts @@ -34,14 +34,30 @@ export type Model = (typeof ALL_MODEL_NAMES)[number]; /** * Resolve a model argument from CLI input to a known Model family. * + * Input is trimmed and lowercased, and one trailing `[...]` context marker + * (for example `claude-sonnet-5[1m]`) is stripped before matching. + * * Precedence rules: * 1. Exact match against ALL_MODEL_NAMES → return as-is. This is the ONLY * path that selects `gpt-5.6-sol` — Sol is intentionally exact-only. + * It is also how to force a model-pinned profile: `--model opus-4-7`, + * `--model sonnet-5`, and so on. * 2. Family heuristics for common variants: * - `gpt-5.4-mini`, `gpt-5.4-turbo`, `gpt-5.4-*` → `gpt-5.4` * - `gpt-*` (anything else GPT, including other 5.6 variants) → `gpt` * - `o3`, `o4`, `o4-mini`, `o1`, `o1-mini`, `o1-pro` → `o-series` - * - `claude-*` (sonnet, opus, haiku, any version) → `claude` + * - Model-pinned Claude families (Opus 4.7, Opus 4.8, Sonnet 5) use an + * allowlist: the bare ID (`claude--`), the bare ID plus + * an 8-digit date snapshot (`-YYYYMMDD`), and the bare ID plus `-latest` + * keep the pinned profile. Any other suffix, including a point release + * with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`, + * `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should + * not read nudges written for an older one. + * - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`, + * because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL + * (lib/eval-model.ts) and moving it would shift every eval baseline. + * The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes. + * - Any other `claude-*` (including `claude-opus-5-5`) → `claude` * - `gemini-*` (2.5-pro, flash, etc.) → `gemini` * 3. Unknown input → returns null (caller decides: error, or fall back). * @@ -50,7 +66,7 @@ export type Model = (typeof ALL_MODEL_NAMES)[number]; * normalizes CLI input to a family name. */ export function resolveModel(input: string): Model | null { - const s = input.trim(); + const s = input.trim().toLowerCase().replace(/\[[^\]]*\]$/, '').trim(); if (!s) return null; // Exact match first @@ -67,10 +83,9 @@ export function resolveModel(input: string): Model | null { if (/^gpt-5\.4(-|$)/.test(s)) return 'gpt-5.4'; if (/^gpt(-|$)/.test(s)) return 'gpt'; if (/^o[0-9]+(-|$)/.test(s)) return 'o-series'; - if (/^claude-opus-4-7(-|$)/.test(s)) return 'opus-4-7'; + const pinned = /^claude-(opus-4-7|opus-4-8|sonnet-5)(-\d{8}|-latest)?$/.exec(s); + if (pinned) return pinned[1] as Model; if (/^claude-fable-5(-|$)/.test(s)) return 'fable-5'; - if (/^claude-opus-4-8(-|$)/.test(s)) return 'opus-4-8'; - if (/^claude-sonnet-5(-|$)/.test(s)) return 'sonnet-5'; if (/^claude(-|$)/.test(s)) return 'claude'; if (/^gemini(-|$)/.test(s)) return 'gemini'; diff --git a/test/model-overlays.test.ts b/test/model-overlays.test.ts index e8ec21d19..452cede5c 100644 --- a/test/model-overlays.test.ts +++ b/test/model-overlays.test.ts @@ -10,6 +10,7 @@ import type { TemplateContext } from '../scripts/resolvers/types'; import { HOST_PATHS } from '../scripts/resolvers/types'; import { generateModelOverlay } from '../scripts/resolvers/model-overlay'; import { resolveModel } from '../scripts/models'; +import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; import { readOverlay } from '../scripts/resolvers/model-overlay'; import { generateCompletenessSection } from '../scripts/resolvers/preamble/generate-completeness-section'; import { generateSetupCommand } from '../scripts/resolvers/utility'; @@ -368,3 +369,47 @@ describe('Sonnet 5 overlay — family nudges', () => { }); }); }); + +describe('model-routing-claude', () => { + test.each([ + ['claude-opus-4-7', 'opus-4-7'], + ['claude-opus-4-7-20260416', 'opus-4-7'], + ['claude-opus-4-7-latest', 'opus-4-7'], + ['claude-opus-4-7-1', 'claude'], + ['claude-opus-4-7-1-20260601', 'claude'], + ['claude-opus-4-8', 'opus-4-8'], + ['claude-opus-4-8-20260701', 'opus-4-8'], + ['claude-opus-4-8-latest', 'opus-4-8'], + ['claude-opus-4-8-1', 'claude'], + ['claude-sonnet-5', 'sonnet-5'], + ['claude-sonnet-5-20260115', 'sonnet-5'], + ['claude-sonnet-5-latest', 'sonnet-5'], + ['claude-sonnet-5-5', 'claude'], + ['claude-sonnet-5-5-20261001', 'claude'], + ['claude-sonnet-5-10', 'claude'], + ['claude-sonnet-5-latest-20260101', 'claude'], + ['claude-opus-5-5', 'claude'], + ['claude-fable-5', 'fable-5'], + ['claude-fable-5-1', 'fable-5'], + ])('%s routes to %s', (input, expected) => { + expect(resolveModel(input)).toBe(expected); + }); + + test('input is lowercased and one trailing [...] marker is stripped', () => { + expect(resolveModel('CLAUDE-SONNET-5')).toBe('sonnet-5'); + expect(resolveModel('claude-sonnet-5[1m]')).toBe('sonnet-5'); + expect(resolveModel('claude-opus-5-5[1m]')).toBe('claude'); + }); + + test('family names force the pinned profile', () => { + expect(resolveModel('opus-4-7')).toBe('opus-4-7'); + expect(resolveModel('opus-4-8')).toBe('opus-4-8'); + expect(resolveModel('sonnet-5')).toBe('sonnet-5'); + }); + + test('Fable prefix exception holds only while the frontier eval model is a Fable 5 point release', () => { + if (resolveModel('claude-fable-5-99') === 'fable-5') { + expect(CLAUDE_FRONTIER_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true); + } + }); +});