fix(models): allowlist routing for model-pinned Claude families (W1)

resolveModel lowercases input and strips one trailing [...] context marker.
Opus 4.7, Opus 4.8 and Sonnet 5 keep their pinned overlay only for the bare
ID, the bare ID plus an 8-digit date, and the bare ID plus -latest; any other
suffix (point releases such as claude-sonnet-5-5, claude-opus-4-7-1) falls to
the generic claude overlay. claude-opus-5-5 routes to claude explicitly (no new
overlay, no new ALL_MODEL_NAMES entry). Fable keeps its prefix match while
CLAUDE_FRONTIER_EVAL_MODEL is claude-fable-5-1, guarded by a test.

Tests: one test/model-overlays.test.ts case per shape in the plan, plus the
lowercase / [1m] marker / family-name cases and the Fable guard.
This commit is contained in:
garrytan committed 2026-10-02 23:40:01 +00:00
1 parent f30b7b788a
commit 2d9d155dd5
2 files changed
+65 -5

No files matched your search

+20 -5
View File
@@ -34,14 +34,30 @@ export type Model = (typeof ALL_MODEL_NAMES)[number];
/**
* Resolve a model argument from CLI input to a known Model family.
*
* Input is trimmed and lowercased, and one trailing `[...]` context marker
* (for example `claude-sonnet-5[1m]`) is stripped before matching.
*
* Precedence rules:
* 1. Exact match against ALL_MODEL_NAMES → return as-is. This is the ONLY
* path that selects `gpt-5.6-sol` — Sol is intentionally exact-only.
* It is also how to force a model-pinned profile: `--model opus-4-7`,
* `--model sonnet-5`, and so on.
* 2. Family heuristics for common variants:
* - `gpt-5.4-mini`, `gpt-5.4-turbo`, `gpt-5.4-*` → `gpt-5.4`
* - `gpt-*` (anything else GPT, including other 5.6 variants) → `gpt`
* - `o3`, `o4`, `o4-mini`, `o1`, `o1-mini`, `o1-pro` → `o-series`
* - `claude-*` (sonnet, opus, haiku, any version) → `claude`
* - Model-pinned Claude families (Opus 4.7, Opus 4.8, Sonnet 5) use an
* allowlist: the bare ID (`claude-<family>-<major>`), the bare ID plus
* an 8-digit date snapshot (`-YYYYMMDD`), and the bare ID plus `-latest`
* keep the pinned profile. Any other suffix, including a point release
* with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`,
* `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should
* not read nudges written for an older one.
* - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`,
* because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL
* (lib/eval-model.ts) and moving it would shift every eval baseline.
* The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes.
* - Any other `claude-*` (including `claude-opus-5-5`) → `claude`
* - `gemini-*` (2.5-pro, flash, etc.) → `gemini`
* 3. Unknown input → returns null (caller decides: error, or fall back).
*
@@ -50,7 +66,7 @@ export type Model = (typeof ALL_MODEL_NAMES)[number];
* normalizes CLI input to a family name.
*/
export function resolveModel(input: string): Model | null {
const s = input.trim();
const s = input.trim().toLowerCase().replace(/\[[^\]]*\]$/, '').trim();
if (!s) return null;
// Exact match first
@@ -67,10 +83,9 @@ export function resolveModel(input: string): Model | null {
if (/^gpt-5\.4(-|$)/.test(s)) return 'gpt-5.4';
if (/^gpt(-|$)/.test(s)) return 'gpt';
if (/^o[0-9]+(-|$)/.test(s)) return 'o-series';
if (/^claude-opus-4-7(-|$)/.test(s)) return 'opus-4-7';
const pinned = /^claude-(opus-4-7|opus-4-8|sonnet-5)(-\d{8}|-latest)?$/.exec(s);
if (pinned) return pinned[1] as Model;
if (/^claude-fable-5(-|$)/.test(s)) return 'fable-5';
if (/^claude-opus-4-8(-|$)/.test(s)) return 'opus-4-8';
if (/^claude-sonnet-5(-|$)/.test(s)) return 'sonnet-5';
if (/^claude(-|$)/.test(s)) return 'claude';
if (/^gemini(-|$)/.test(s)) return 'gemini';
+45
View File
@@ -10,6 +10,7 @@ import type { TemplateContext } from '../scripts/resolvers/types';
import { HOST_PATHS } from '../scripts/resolvers/types';
import { generateModelOverlay } from '../scripts/resolvers/model-overlay';
import { resolveModel } from '../scripts/models';
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
import { readOverlay } from '../scripts/resolvers/model-overlay';
import { generateCompletenessSection } from '../scripts/resolvers/preamble/generate-completeness-section';
import { generateSetupCommand } from '../scripts/resolvers/utility';
@@ -368,3 +369,47 @@ describe('Sonnet 5 overlay — family nudges', () => {
});
});
});
describe('model-routing-claude', () => {
test.each([
['claude-opus-4-7', 'opus-4-7'],
['claude-opus-4-7-20260416', 'opus-4-7'],
['claude-opus-4-7-latest', 'opus-4-7'],
['claude-opus-4-7-1', 'claude'],
['claude-opus-4-7-1-20260601', 'claude'],
['claude-opus-4-8', 'opus-4-8'],
['claude-opus-4-8-20260701', 'opus-4-8'],
['claude-opus-4-8-latest', 'opus-4-8'],
['claude-opus-4-8-1', 'claude'],
['claude-sonnet-5', 'sonnet-5'],
['claude-sonnet-5-20260115', 'sonnet-5'],
['claude-sonnet-5-latest', 'sonnet-5'],
['claude-sonnet-5-5', 'claude'],
['claude-sonnet-5-5-20261001', 'claude'],
['claude-sonnet-5-10', 'claude'],
['claude-sonnet-5-latest-20260101', 'claude'],
['claude-opus-5-5', 'claude'],
['claude-fable-5', 'fable-5'],
['claude-fable-5-1', 'fable-5'],
])('%s routes to %s', (input, expected) => {
expect(resolveModel(input)).toBe(expected);
});
test('input is lowercased and one trailing [...] marker is stripped', () => {
expect(resolveModel('CLAUDE-SONNET-5')).toBe('sonnet-5');
expect(resolveModel('claude-sonnet-5[1m]')).toBe('sonnet-5');
expect(resolveModel('claude-opus-5-5[1m]')).toBe('claude');
});
test('family names force the pinned profile', () => {
expect(resolveModel('opus-4-7')).toBe('opus-4-7');
expect(resolveModel('opus-4-8')).toBe('opus-4-8');
expect(resolveModel('sonnet-5')).toBe('sonnet-5');
});
test('Fable prefix exception holds only while the frontier eval model is a Fable 5 point release', () => {
if (resolveModel('claude-fable-5-99') === 'fable-5') {
expect(CLAUDE_FRONTIER_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true);
}
});
});