fix(eval-plan): pilot start keeps the R54 refusal; --tier-question rejects a flag

Review pass 1 (full depth) found the frontier refusal could be skipped by
starting a pilot on Haiku or Sonnet arms behind a dry run made on other arms,
or behind a dry-run.json written before the check existed. startPilot now
refuses a pilot whose every arm is Haiku or Sonnet unless its dry run recorded
a tier question. dry-run --tier-question no longer takes the next flag as its
answer. The testing doc now says the frontier re-cost is printed for
comparison with the brief cap, not enforced.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

bin/gstack-eval-plan

@@ -165,7 +165,9 @@ function doDryRun(rest: string[]): number {
   if (!arms.length) return usage('dry-run needs --arms <model,model>');
   const scope = scopeOf(rest);
   const modelRules = lintModels(rest, arms.map(model => ({ model, scope, where: 'dry-run --arms' })), 'dry-run');
-  const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules, tierQuestion: flag(rest, '--tier-question') });
+  const tierQuestion = flag(rest, '--tier-question');
+  if (tierQuestion?.startsWith('--')) return usage('--tier-question needs the question the preregistration asks about that tier, not another flag');
+  const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules, tierQuestion });
   const out = flag(rest, '--out');
   if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); }
   if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r));

changelog.d/garrytan-frontier-models-under-test.md

@@ -1,3 +1,3 @@
 ### Changed
 
-- Evals test frontier models (R54). /eval-plan and docs/TESTING_INTERNALS.md#frontier-models-under-test now say that models under test are Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary. Haiku and Sonnet aren't stand-ins for a frontier model, and a Haiku- or Sonnet-only result can't carry a KEEP, FIX-METHOD or CUT-CANDIDATE verdict. Judges, review voices and housekeeping models keep their models. `gstack-eval-plan dry-run` refuses a plan whose every arm is Haiku or Sonnet unless `--tier-question` names what it asks about that tier, warns on a mixed plan, and prices the whole plan on frontier models.
+- Evals test frontier models (R54). /eval-plan and docs/TESTING_INTERNALS.md#frontier-models-under-test now say that models under test are Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary. Haiku and Sonnet aren't stand-ins for a frontier model, and a Haiku- or Sonnet-only result can't carry a KEEP, FIX-METHOD or CUT-CANDIDATE verdict. Judges, review voices and housekeeping models keep their models. `gstack-eval-plan dry-run` refuses a plan whose every arm is Haiku or Sonnet unless `--tier-question` names what it asks about that tier, warns on a mixed plan, and prices the whole plan on frontier models. `pilot start` refuses a Haiku- or Sonnet-only pilot unless its dry run recorded that tier question.

docs/TESTING_INTERNALS.md

@@ -1783,9 +1783,11 @@ model they already use.
 **The check.** `gstack-eval-plan dry-run` prints `CHECK: frontier` whenever an arm
 is Haiku or Sonnet. When every arm is one, the dry run fails unless
 `--tier-question "<what the preregistration asks about that tier>"` is passed.
-When only some arms are, it warns. Either way it prices the whole plan on
-`claude-opus-5-5` and `gpt-6.1-sol`, so the budget is checked against frontier
-prices before any arm is swapped.
+When only some arms are, it warns. Either way it prints the whole plan's price
+on `claude-opus-5-5` and `gpt-6.1-sol`. The dry run doesn't enforce that number:
+compare it with the brief cap before any arm is swapped. `pilot start` refuses a
+pilot whose every arm is Haiku or Sonnet unless its dry run recorded a tier
+question.
 
 ## Running evals as an agent: detach
 

lib/eval-plan.ts

@@ -390,6 +390,7 @@ export interface PilotStartOptions { out: string; cap: number; n: number; arms:
 export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: { code: ResultCodeName; message: string } } {
   const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult;
   if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } };
+  if (o.arms.length && o.arms.every(a => isSmallTierModel(armModel(a))) && !dry.checks.frontier?.tier_question) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `every pilot arm is Haiku or Sonnet and ${o.dryRun} records no --tier-question (R54, docs/TESTING_INTERNALS.md#frontier-models-under-test)` } };
   const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget };
   const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' };
   const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`;

test/pricing-eval-plan.test.ts

@@ -133,6 +133,28 @@ describe('dry run', () => {
     expect(frontier.checks.frontier).toMatchObject({ ok: true, small_tier: [] });
     expect(renderDryRun(frontier)).not.toContain('CHECK: frontier');
     expect(['claude-opus-5-5', 'gpt-6.1-sol', 'gpt-6-luna', 'gemini-3.8-flash'].some(isSmallTierModel)).toBe(false);
+    const dryFile = (name: string, r: unknown) => { const f = path.join(d, name); fs.writeFileSync(f, JSON.stringify(r)); return f; };
+    const swapped = startPilot({ out: path.join(d, 'swapped'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('frontier.json', frontier) });
+    expect(swapped.error?.code).toBe('EVAL_DRY_RUN_FAILED');
+    expect(fs.existsSync(path.join(d, 'swapped'))).toBe(false);
+    expect(startPilot({ out: path.join(d, 'asked'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('asked.json', asked) }).error).toBeUndefined();
+    expect(startPilot({ out: path.join(d, 'mixed'), cap: 5, n: 1, arms: ['claude-opus-5-5', 'claude-sonnet-5-5'], dryRun: dryFile('frontier2.json', frontier) }).error).toBeUndefined();
+  });
+
+  test('R54 CLI: --tier-question refuses another flag as its answer', () => {
+    const host = capyHost();
+    try {
+      const rows = path.join(host.base, 'rows.jsonl'); fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello world' }) + '\n');
+      const retry = path.join(host.base, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
+      const run = (...extra: string[]) => spawnSync(BIN, ['dry-run', '--input', rows, '--arms', 'claude-haiku-4-5', '--retry-prompt', retry, ...extra], { cwd: host.base, env: host.env, encoding: 'utf8', timeout: 30_000 });
+      const flagAnswer = run('--tier-question', '--json');
+      expect(flagAnswer.status).not.toBe(0);
+      expect(flagAnswer.stderr).toContain('--tier-question needs the question');
+      const refused = run();
+      expect(refused.status).not.toBe(0);
+      expect(refused.stderr).toContain('R54');
+      expect(run('--tier-question', 'does the diet hold on Haiku').status).toBe(0);
+    } finally { host.cleanup(); }
   });
 });