diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 03910b7b2..6e2b90bb5 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -165,7 +165,9 @@ function doDryRun(rest: string[]): number { if (!arms.length) return usage('dry-run needs --arms '); const scope = scopeOf(rest); const modelRules = lintModels(rest, arms.map(model => ({ model, scope, where: 'dry-run --arms' })), 'dry-run'); - const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules, tierQuestion: flag(rest, '--tier-question') }); + const tierQuestion = flag(rest, '--tier-question'); + if (tierQuestion?.startsWith('--')) return usage('--tier-question needs the question the preregistration asks about that tier, not another flag'); + const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules, tierQuestion }); const out = flag(rest, '--out'); if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); } if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r)); diff --git a/changelog.d/garrytan-frontier-models-under-test.md b/changelog.d/garrytan-frontier-models-under-test.md index 46af042ba..e631d1ac8 100644 --- a/changelog.d/garrytan-frontier-models-under-test.md +++ b/changelog.d/garrytan-frontier-models-under-test.md @@ -1,3 +1,3 @@ ### Changed -- Evals test frontier models (R54). /eval-plan and docs/TESTING_INTERNALS.md#frontier-models-under-test now say that models under test are Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary. Haiku and Sonnet aren't stand-ins for a frontier model, and a Haiku- or Sonnet-only result can't carry a KEEP, FIX-METHOD or CUT-CANDIDATE verdict. Judges, review voices and housekeeping models keep their models. `gstack-eval-plan dry-run` refuses a plan whose every arm is Haiku or Sonnet unless `--tier-question` names what it asks about that tier, warns on a mixed plan, and prices the whole plan on frontier models. +- Evals test frontier models (R54). /eval-plan and docs/TESTING_INTERNALS.md#frontier-models-under-test now say that models under test are Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary. Haiku and Sonnet aren't stand-ins for a frontier model, and a Haiku- or Sonnet-only result can't carry a KEEP, FIX-METHOD or CUT-CANDIDATE verdict. Judges, review voices and housekeeping models keep their models. `gstack-eval-plan dry-run` refuses a plan whose every arm is Haiku or Sonnet unless `--tier-question` names what it asks about that tier, warns on a mixed plan, and prices the whole plan on frontier models. `pilot start` refuses a Haiku- or Sonnet-only pilot unless its dry run recorded that tier question. diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index e608b043a..ae116b469 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1783,9 +1783,11 @@ model they already use. **The check.** `gstack-eval-plan dry-run` prints `CHECK: frontier` whenever an arm is Haiku or Sonnet. When every arm is one, the dry run fails unless `--tier-question ""` is passed. -When only some arms are, it warns. Either way it prices the whole plan on -`claude-opus-5-5` and `gpt-6.1-sol`, so the budget is checked against frontier -prices before any arm is swapped. +When only some arms are, it warns. Either way it prints the whole plan's price +on `claude-opus-5-5` and `gpt-6.1-sol`. The dry run doesn't enforce that number: +compare it with the brief cap before any arm is swapped. `pilot start` refuses a +pilot whose every arm is Haiku or Sonnet unless its dry run recorded a tier +question. ## Running evals as an agent: detach diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 3f5325e04..95d21fd4b 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -390,6 +390,7 @@ export interface PilotStartOptions { out: string; cap: number; n: number; arms: export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: { code: ResultCodeName; message: string } } { const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult; if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } }; + if (o.arms.length && o.arms.every(a => isSmallTierModel(armModel(a))) && !dry.checks.frontier?.tier_question) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `every pilot arm is Haiku or Sonnet and ${o.dryRun} records no --tier-question (R54, docs/TESTING_INTERNALS.md#frontier-models-under-test)` } }; const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget }; const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`; diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index 32df382c7..fd2611b4b 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -133,6 +133,28 @@ describe('dry run', () => { expect(frontier.checks.frontier).toMatchObject({ ok: true, small_tier: [] }); expect(renderDryRun(frontier)).not.toContain('CHECK: frontier'); expect(['claude-opus-5-5', 'gpt-6.1-sol', 'gpt-6-luna', 'gemini-3.8-flash'].some(isSmallTierModel)).toBe(false); + const dryFile = (name: string, r: unknown) => { const f = path.join(d, name); fs.writeFileSync(f, JSON.stringify(r)); return f; }; + const swapped = startPilot({ out: path.join(d, 'swapped'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('frontier.json', frontier) }); + expect(swapped.error?.code).toBe('EVAL_DRY_RUN_FAILED'); + expect(fs.existsSync(path.join(d, 'swapped'))).toBe(false); + expect(startPilot({ out: path.join(d, 'asked'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('asked.json', asked) }).error).toBeUndefined(); + expect(startPilot({ out: path.join(d, 'mixed'), cap: 5, n: 1, arms: ['claude-opus-5-5', 'claude-sonnet-5-5'], dryRun: dryFile('frontier2.json', frontier) }).error).toBeUndefined(); + }); + + test('R54 CLI: --tier-question refuses another flag as its answer', () => { + const host = capyHost(); + try { + const rows = path.join(host.base, 'rows.jsonl'); fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello world' }) + '\n'); + const retry = path.join(host.base, 'retry.txt'); fs.writeFileSync(retry, 'retry\n'); + const run = (...extra: string[]) => spawnSync(BIN, ['dry-run', '--input', rows, '--arms', 'claude-haiku-4-5', '--retry-prompt', retry, ...extra], { cwd: host.base, env: host.env, encoding: 'utf8', timeout: 30_000 }); + const flagAnswer = run('--tier-question', '--json'); + expect(flagAnswer.status).not.toBe(0); + expect(flagAnswer.stderr).toContain('--tier-question needs the question'); + const refused = run(); + expect(refused.status).not.toBe(0); + expect(refused.stderr).toContain('R54'); + expect(run('--tier-question', 'does the diet hold on Haiku').status).toBe(0); + } finally { host.cleanup(); } }); });