feat(eval-plan): prereg diagnostics dry run (W8) and .gstack/eval-models.json lint (W9), WIP

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
(cherry picked from commit b509096218)

bin/gstack-eval-plan

@@ -2,7 +2,7 @@
 /**
  * gstack-eval-plan — the tier-1 machinery behind /eval-plan (plan E1).
  *
- *   gstack-eval-plan prereg --check <file> | --init <file> [--title <t>]
+ *   gstack-eval-plan prereg --check <file> [--allow-repo-commands] [--json] | --init <file> [--title <t>]
  *   gstack-eval-plan arms [--arms <model,model>] [--policy-doc <file>] [--input-tokens N] [--output-tokens N] [--json]
  *   gstack-eval-plan dry-run --input <rows.jsonl> --arms <model,model> [--fields a,b] [--output-budget N]
  *                            [--retry-prompt <file>] [--max-rows N] [--out <dir>] [--json]
@@ -29,7 +29,8 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { spawnSync } from 'node:child_process';
-import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan';
+import { checkPrereg, DEFAULT_TOKENS, renderPreregDiagnostics, runPreregDiagnostics, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan';
+import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models';
 import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
 import { resultLine } from '../lib/result-codes';
 import { resolveBaseBranch } from '../lib/ship-policy';
@@ -37,7 +38,10 @@ import { SpendCapExceeded } from '../lib/spend-ledger';
 
 const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md)
 
-  prereg --check <file>              the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure
+  prereg --check <file> [--allow-repo-commands] [--json]
+                                     the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure;
+                                     an optional Diagnostics section's summarizer runs on its synthetic fixture only with
+                                     --allow-repo-commands (else it is listed and exit 3) and every count must match
   prereg --init <file> [--title <t>] write the template
   arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json]
                                      arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item
@@ -68,6 +72,24 @@ function num(args: string[], name: string, fallback?: number): number | undefine
 function usage(message: string): number { process.stderr.write(`gstack-eval-plan: ${message}\n${HELP}\n`); return EXIT.usage; }
 const csv = (v: string | undefined) => (v ?? '').split(',').map(s => s.trim()).filter(Boolean);
 
+function scopeOf(args: string[]): EvalScope {
+  const v = flag(args, '--scope') ?? 'full';
+  if (!(EVAL_SCOPES as readonly string[]).includes(v)) throw new Error(`--scope needs one of ${EVAL_SCOPES.join(', ')}`);
+  return v as EvalScope;
+}
+
+/** Lint arms against .gstack/eval-models.json (origin/<base>, or the working tree with --models-from head) and print its lines. */
+function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScope; where: string }>, command: string): ModelRulesCheck {
+  const from = flag(args, '--models-from') ?? 'base';
+  if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head');
+  const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 });
+  const repoRoot = top.status === 0 ? top.stdout.trim() : null;
+  const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' });
+  if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n');
+  for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n');
+  return check;
+}
+
 function doPrereg(rest: string[]): number {
   const init = flag(rest, '--init');
   if (init) {
@@ -80,18 +102,33 @@ function doPrereg(rest: string[]): number {
   const file = flag(rest, '--check');
   if (!file || !fs.existsSync(file)) return usage('prereg needs --check <existing file> or --init <file>');
   const r = checkPrereg(fs.readFileSync(file, 'utf8'));
-  if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r }) + '\n');
-  else process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''} file=${file}\n`);
-  if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`)].join('; ')) + '\n'); return EXIT.fail; }
+  const d = r.diagnostics;
+  const runnable = d && !d.errors.length;
+  const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: path.dirname(path.resolve(file)), encoding: 'utf8', timeout: 30_000 });
+  const repoRoot = top.status === 0 ? top.stdout.trim() : path.dirname(path.resolve(file));
+  const allowed = rest.includes('--allow-repo-commands');
+  const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null;
+  if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n');
+  else {
+    process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`);
+    if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`);
+    if (run) process.stdout.write(renderPreregDiagnostics(run));
+  }
+  if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n');
+  if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n');
+  if (!r.ok || (run && !run.ok)) return EXIT.fail;
+  if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; }
   return EXIT.ok;
 }
 
 function doArms(rest: string[]): number {
   const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! };
+  const scope = scopeOf(rest);
   const r = resolveArms({ cwd: process.cwd(), explicit: csv(flag(rest, '--arms')), policyDoc: flag(rest, '--policy-doc'), tokens });
   if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r, tokens }) + '\n'); else process.stdout.write(renderArms(r, tokens));
   for (const e of r.errors) process.stderr.write(resultLine('gstack-eval-plan arms', e.code, e.message) + '\n');
-  return r.errors.length ? EXIT.fail : EXIT.ok;
+  const rules = r.arms.length ? lintModels(rest, r.arms.map(a => ({ model: a.model, scope, where: a.source })), 'arms') : null;
+  return r.errors.length || (rules && !rules.ok) ? EXIT.fail : EXIT.ok;
 }
 
 function doDryRun(rest: string[]): number {
@@ -99,13 +136,15 @@ function doDryRun(rest: string[]): number {
   const arms = csv(flag(rest, '--arms'));
   if (!input || !fs.existsSync(input)) return usage('dry-run needs --input <existing rows.jsonl>');
   if (!arms.length) return usage('dry-run needs --arms <model,model>');
-  const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap') });
+  const scope = scopeOf(rest);
+  const modelRules = lintModels(rest, arms.map(model => ({ model, scope, where: 'dry-run --arms' })), 'dry-run');
+  const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules });
   const out = flag(rest, '--out');
   if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); }
   if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r));
   if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`);
   if (!r.pass) {
-    const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : ''].filter(Boolean).join('; ');
+    const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : '', r.checks.model_rules && !r.checks.model_rules.ok ? 'model rules refused an arm (EVAL_MODEL lines above)' : ''].filter(Boolean).join('; ');
     process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n');
     return EXIT.fail;
   }
@@ -119,6 +158,8 @@ function doPilot(rest: string[]): number {
   if (sub === 'start') {
     const cap = num(args, '--cap'); const arms = csv(flag(args, '--arms')); const dry = flag(args, '--dry-run');
     if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap <usd> --arms <m,m> --dry-run <existing dry-run.json>');
+    const rules = lintModels(args, arms.map(model => ({ model, scope: scopeOf(args), where: 'pilot start --arms' })), 'pilot start');
+    if (!rules.ok) return EXIT.fail;
     const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry });
     if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; }
     for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`);
@@ -150,7 +191,7 @@ function doPilot(rest: string[]): number {
   return usage(`unknown pilot command ${sub ?? ''}`);
 }
 
-const PRICE_GATE_FLAGS = new Set(['--gate', '--gate-file', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out']);
+const PRICE_GATE_FLAGS = new Set(['--gate', '--gate-file', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out', '--models-from']);
 
 function doPriceGate(rest: string[]): number {
   for (let i = 0; i < rest.length; i++) {
@@ -171,6 +212,7 @@ function doPriceGate(rest: string[]): number {
   const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) });
   if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; }
   const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') });
+  if (r.receipt?.rows.length && !lintModels(rest, r.receipt.rows.map(row => ({ model: row.model, scope: row.scope, where: `purpose=${row.purpose}` })), 'price-gate').ok) return EXIT.fail;
   if (r.receipt) {
     const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate'));
     const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`);

lib/eval-models.ts

@@ -0,0 +1,160 @@
+/**
+ * eval-models — a repo's eval model rules, .gstack/eval-models.json, and the
+ * lint gstack-eval-plan runs over every arm it prices or reserves (field
+ * list 2, W9). The file states the rules as data, never prose:
+ *
+ *   allow     ids or globs an arm may use; required, and an id it does not
+ *             match fails closed (an uncatalogued or misspelled model is
+ *             never admitted by default)
+ *   deny      ids or globs no arm may use, checked first
+ *   scopes    per eval scope (smoke, full: the call-plan row scope; an arm
+ *             with no stated scope is full) an extra allow and/or deny list,
+ *             so "Fable in smoke only" is scopes.full.deny ["claude-fable-*"]
+ *   families  optional ordered map, newest first: an arm must be the newest
+ *             id of its family, and with the map present an arm in no family
+ *             fails as unresolved instead of passing unchecked
+ *
+ * Literal allow/deny apply whatever lib/model-catalog.ts holds. MODEL_CATALOG
+ * (one current model per provider and review tier) only checks the families
+ * map's currency: a catalog model listed after another id in its family is a
+ * stale map. The reviewed copy is read from origin/<base>; the working tree's
+ * copy only with --models-from head. Doc: docs/eval-models.md.
+ */
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { MODEL_CATALOG } from './model-catalog';
+import type { ResultCodeName } from './result-codes';
+
+export const EVAL_MODELS_PATH = '.gstack/eval-models.json';
+export const EVAL_SCOPES = ['smoke', 'full'] as const;
+export type EvalScope = typeof EVAL_SCOPES[number];
+export interface ScopeRules { allow?: string[]; deny: string[] }
+export interface EvalModelRules { schema_version: 1; allow: string[]; deny: string[]; scopes: Partial<Record<EvalScope, ScopeRules>>; families: Record<string, string[]> }
+export interface ArmUse { model: string; scope: EvalScope; where: string }
+export type ModelVerdict = 'ok' | 'denied' | 'old-generation' | 'generation-unresolved';
+export interface ModelFinding extends ArmUse { verdict: ModelVerdict; reason: string }
+export interface ModelRulesCheck { ok: boolean; source: string; lines: string[]; findings: ModelFinding[]; errors: Array<{ code: ResultCodeName; message: string }> }
+
+const PATTERN = /^[A-Za-z0-9._:/*?-]+$/;
+const TOP_KEYS = ['schema_version', 'allow', 'deny', 'scopes', 'families'];
+
+function patterns(v: unknown, at: string, errors: string[], o: { required?: boolean; literal?: boolean } = {}): string[] {
+  if (v === undefined && !o.required) return [];
+  if (!Array.isArray(v) || (o.required && v.length === 0)) { errors.push(`${at} must be ${o.required ? 'a non-empty' : 'an'} array of model ids${o.literal ? '' : ' or globs'}`); return []; }
+  const out: string[] = [];
+  v.forEach((p, i) => {
+    if (typeof p !== 'string' || !PATTERN.test(p) || (o.literal && /[*?]/.test(p))) errors.push(`${at}[${i}] ${JSON.stringify(p)} is not a model id${o.literal ? ' (family entries are literal ids, newest first)' : ' or glob (letters, digits, . _ : / - and the wildcards * ?)'}`);
+    else out.push(p);
+  });
+  return out;
+}
+
+const isObject = (v: unknown): v is Record<string, unknown> => !!v && typeof v === 'object' && !Array.isArray(v);
+
+/** Parse and validate the rules; any unknown key, scope or malformed entry fails the whole file. */
+export function parseEvalModels(text: string): { rules: EvalModelRules } | { errors: string[] } {
+  let doc: unknown;
+  try { doc = JSON.parse(text); } catch (e: any) { return { errors: [`not valid JSON: ${e.message}`] }; }
+  if (!isObject(doc)) return { errors: ['must be a JSON object'] };
+  const errors: string[] = [];
+  for (const k of Object.keys(doc)) if (!TOP_KEYS.includes(k)) errors.push(`unknown key ${JSON.stringify(k)} (known: ${TOP_KEYS.join(', ')})`);
+  if (doc.schema_version !== 1) errors.push('schema_version must be 1');
+  const rules: EvalModelRules = { schema_version: 1, allow: patterns(doc.allow, 'allow', errors, { required: true }), deny: patterns(doc.deny, 'deny', errors), scopes: {}, families: {} };
+  if (doc.scopes !== undefined && !isObject(doc.scopes)) errors.push('scopes must be an object keyed by eval scope');
+  for (const [scope, v] of Object.entries(isObject(doc.scopes) ? doc.scopes : {})) {
+    if (!(EVAL_SCOPES as readonly string[]).includes(scope)) { errors.push(`scopes.${scope}: unknown eval scope (known: ${EVAL_SCOPES.join(', ')})`); continue; }
+    if (!isObject(v)) { errors.push(`scopes.${scope} must be an object with allow and/or deny`); continue; }
+    const extra = Object.keys(v).filter(k => k !== 'allow' && k !== 'deny');
+    if (extra.length) errors.push(`scopes.${scope}: unknown key(s) ${extra.join(', ')} (known: allow, deny)`);
+    rules.scopes[scope as EvalScope] = { ...(v.allow !== undefined ? { allow: patterns(v.allow, `scopes.${scope}.allow`, errors, { required: true }) } : {}), deny: patterns(v.deny, `scopes.${scope}.deny`, errors) };
+  }
+  if (doc.families !== undefined && !isObject(doc.families)) errors.push('families must be an object of family name to model ids, newest first');
+  const owner = new Map<string, string>();
+  for (const [family, ids] of Object.entries(isObject(doc.families) ? doc.families : {})) {
+    rules.families[family] = patterns(ids, `families.${family}`, errors, { required: true, literal: true });
+    for (const id of rules.families[family]!) {
+      if (owner.has(id)) errors.push(`families: ${id} is listed in both ${owner.get(id)} and ${family}`);
+      else owner.set(id, family);
+    }
+  }
+  for (const entry of MODEL_CATALOG) {
+    const family = owner.get(entry.model);
+    const head = family ? rules.families[family]![0] : entry.model;
+    if (head !== entry.model) errors.push(`families.${family} lists ${head} before ${entry.model}, gstack's current ${entry.provider} ${entry.tier} model (lib/model-catalog.ts, verified ${entry.verifiedAt}); list newest first`);
+  }
+  return errors.length ? { errors } : { rules };
+}
+
+/** The id as given, without a provider prefix, and a Claude id without its date suffix: the forms a rule may name. */
+function forms(model: string): string[] {
+  const bare = model.replace(/^[a-z-]+[/:]/, '');
+  return [...new Set([model, bare, /^claude-/.test(bare) ? bare.replace(/-\d{8}$/, '') : bare])];
+}
+const glob = (p: string) => new RegExp(`^${p.split('').map(c => (c === '*' ? '.*' : c === '?' ? '.' : c.replace(/[.+^${}()|[\]\\]/g, '\\$&'))).join('')}$`);
+const firstMatch = (list: string[] | undefined, model: string) => (list ?? []).find(p => forms(model).some(f => glob(p).test(f)));
+
+/** Deny, the scope's deny, allow, the scope's allow, then the newest-generation rule; the first that refuses decides. */
+export function lintArmModels(rules: EvalModelRules, uses: ArmUse[]): ModelFinding[] {
+  const families = Object.entries(rules.families);
+  return uses.map((use): ModelFinding => {
+    const scope = rules.scopes[use.scope];
+    const denied = firstMatch(rules.deny, use.model);
+    if (denied) return { ...use, verdict: 'denied', reason: `deny ${denied}` };
+    const scopeDenied = firstMatch(scope?.deny, use.model);
+    if (scopeDenied) return { ...use, verdict: 'denied', reason: `scopes.${use.scope}.deny ${scopeDenied}` };
+    const allowed = firstMatch(rules.allow, use.model);
+    if (!allowed) return { ...use, verdict: 'denied', reason: 'matches no allow entry (unknown ids fail closed)' };
+    if (scope?.allow && !firstMatch(scope.allow, use.model)) return { ...use, verdict: 'denied', reason: `matches no scopes.${use.scope}.allow entry` };
+    if (!families.length) return { ...use, verdict: 'ok', reason: `allow ${allowed}` };
+    const family = families.find(([, ids]) => ids.some(id => forms(use.model).includes(id)));
+    if (!family) return { ...use, verdict: 'generation-unresolved', reason: 'in no families entry, so the newest-generation rule cannot be checked' };
+    if (!forms(use.model).includes(family[1][0]!)) return { ...use, verdict: 'old-generation', reason: `families.${family[0]} newest is ${family[1][0]}` };
+    return { ...use, verdict: 'ok', reason: `allow ${allowed}; newest in families.${family[0]}` };
+  });
+}
+
+const VERDICT_CODE: Record<Exclude<ModelVerdict, 'ok'>, ResultCodeName> = { denied: 'EVAL_MODEL_DENIED', 'old-generation': 'EVAL_MODEL_OLD_GENERATION', 'generation-unresolved': 'EVAL_MODEL_GENERATION_UNRESOLVED' };
+
+function gitText(repoRoot: string, args: string[]): string | null {
+  const r = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 30_000 });
+  return r.status === 0 ? r.stdout : null;
+}
+
+export interface EvalModelsLoad { text: string | null; label: string; notes: string[]; error: string | null }
+
+/** The reviewed rules from origin/<base> (default) or the working tree's copy; text null when none are declared. */
+export function loadEvalModels(repoRoot: string | null, o: { from: 'base' | 'head'; base: string }): EvalModelsLoad {
+  if (!repoRoot) return { text: null, label: 'none (not a git repository)', notes: [], error: null };
+  const headFile = path.join(repoRoot, EVAL_MODELS_PATH);
+  const head = fs.existsSync(headFile) ? fs.readFileSync(headFile, 'utf8') : null;
+  if (o.from === 'head') return { text: head, label: head === null ? `none (no ${EVAL_MODELS_PATH} in the working tree)` : `head (unreviewed) ${EVAL_MODELS_PATH}`, notes: [], error: null };
+  const ref = `origin/${o.base.replace(/^origin\//, '')}`;
+  const sha = gitText(repoRoot, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`])?.trim();
+  if (!sha) {
+    if (head === null) return { text: null, label: `none (${ref} is not a resolvable commit)`, notes: [], error: null };
+    return { text: null, label: ref, notes: [], error: `${ref} is not a resolvable commit, so the reviewed ${EVAL_MODELS_PATH} cannot be read; fetch it, or pass --models-from head to lint with the working-tree copy` };
+  }
+  const at = `${ref}@${sha.slice(0, 12)}`;
+  const base = gitText(repoRoot, ['show', `${sha}:${EVAL_MODELS_PATH}`]);
+  if (base === null) return { text: null, label: `none at ${at}`, notes: head === null ? [] : [`the working tree has an unreviewed ${EVAL_MODELS_PATH}; --models-from head lints with it`], error: null };
+  return { text: base, label: `${at}:${EVAL_MODELS_PATH}`, notes: head !== null && head !== base ? [`the working-tree ${EVAL_MODELS_PATH} differs from the reviewed copy; linting with ${ref}`] : [], error: null };
+}
+
+/** Load, validate and lint: one EVAL_MODEL line per arm, then the EVAL_MODELS summary; a refusal or an invalid file is an error. */
+export function checkModelRules(repoRoot: string | null, uses: ArmUse[], o: { from: 'base' | 'head'; base: string }): ModelRulesCheck {
+  const load = loadEvalModels(repoRoot, o);
+  const notes = load.notes.map(n => `EVAL_MODELS_NOTE: ${n}`);
+  const invalid = (message: string): ModelRulesCheck => ({ ok: false, source: load.label, lines: [...notes, `EVAL_MODELS: invalid source=${load.label}`], findings: [], errors: [{ code: 'EVAL_MODELS_INVALID', message }] });
+  if (load.error) return invalid(load.error);
+  if (load.text === null) return { ok: true, source: load.label, lines: [...notes, `EVAL_MODELS: ${load.label}`], findings: [], errors: [] };
+  const parsed = parseEvalModels(load.text);
+  if ('errors' in parsed) return invalid(`${load.label}: ${parsed.errors.join('; ')}`);
+  const findings = lintArmModels(parsed.rules, uses);
+  const bad = findings.filter(f => f.verdict !== 'ok');
+  return {
+    ok: bad.length === 0, source: load.label, findings,
+    lines: [...notes, ...findings.map(f => `EVAL_MODEL: ${f.model} scope=${f.scope} ${f.verdict} (${f.reason}) where=${f.where}`), `EVAL_MODELS: ${bad.length ? `fail refused=${bad.length}/${findings.length}` : `ok n=${findings.length}`} source=${load.label}`],
+    errors: bad.map(f => ({ code: VERDICT_CODE[f.verdict as Exclude<ModelVerdict, 'ok'>], message: `${f.model} (scope ${f.scope}, ${f.where}): ${f.reason}` })),
+  };
+}

lib/eval-plan.ts

@@ -17,13 +17,14 @@ import { modelsMain } from './model-policy-cli';
 import { isConfigUnavailable, loadConfigFromBase } from './pregate/config';
 import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pricingRow } from './pricing';
 import type { ResultCodeName } from './result-codes';
+import type { ModelRulesCheck } from './eval-models';
 import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger';
 
 // ---------------------------------------------------------------------------
 // preregistration
 // ---------------------------------------------------------------------------
 export const PREREG_SECTIONS = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'] as const;
-export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[] }
+export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; diagnostics: PreregDiagnostics | null }
 
 export function preregTemplate(title = 'Eval preregistration'): string {
   return `# ${title}
@@ -42,22 +43,103 @@ export function preregTemplate(title = 'Eval preregistration'): string {
 
 ## Held-out exposure
 <!-- Every held-out set this plan touches: | set | prior exposure (which runs, which models) | reservation (what this run may read) | -->
+<!-- Optional: a \`## Diagnostics\` section declares the summarizer (\`summarizer: <command naming {fixture}>\`), its synthetic records (\`fixture: <path>\`) and one | diagnostic | expected | row per diagnostic; \`prereg --check --allow-repo-commands\` runs it. -->
 `;
 }
 
-export function checkPrereg(text: string): PreregCheck {
+/** A section's content lines (blank lines and one-line `<!-- -->` guidance dropped); null when the heading is absent. */
+function sectionBody(text: string, name: string): string[] | null {
   const lines = text.split('\n');
   const headings = lines.map((l, i) => ({ i, m: /^#{1,6}\s+(.+?)\s*$/.exec(l) })).filter(x => x.m) as Array<{ i: number; m: RegExpExecArray }>;
+  const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, ''));
+  if (idx < 0) return null;
+  const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length;
+  return lines.slice(headings[idx]!.i + 1, end).filter(l => l.trim() && !/^\s*<!--.*-->\s*$/.test(l));
+}
+
+export function checkPrereg(text: string): PreregCheck {
   const present: string[] = []; const missing: string[] = []; const empty: string[] = [];
   for (const name of PREREG_SECTIONS) {
-    const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, ''));
-    if (idx < 0) { missing.push(name); continue; }
-    const start = headings[idx]!.i + 1;
-    const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length;
-    const body = lines.slice(start, end).filter(l => l.trim() && !/^\s*<!--.*-->\s*$/.test(l));
-    if (body.length === 0) empty.push(name); else present.push(name);
+    const body = sectionBody(text, name);
+    if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name);
   }
-  return { ok: missing.length === 0 && empty.length === 0, present, missing, empty };
+  const diagnostics = parsePreregDiagnostics(text);
+  return { ok: missing.length === 0 && empty.length === 0 && !diagnostics?.errors.length, present, missing, empty, diagnostics };
+}
+
+// ---------------------------------------------------------------------------
+// preregistered diagnostics: the declared summarizer on a declared synthetic fixture
+// ---------------------------------------------------------------------------
+/** Placeholder the summarizer command names the fixture by; it runs as "$GSTACK_PREREG_FIXTURE", never spliced into the shell text. */
+export const PREREG_FIXTURE_PLACEHOLDER = '{fixture}';
+export const PREREG_SUMMARIZER_TIMEOUT_MS = 300_000;
+export interface PreregDiagnostics { summarizer: string | null; fixture: string | null; expected: Array<{ name: string; count: number }>; errors: string[] }
+export interface DiagnosticRow { name: string; expected: number | null; got: number | null; status: 'ok' | 'mismatch' | 'missing' | 'undeclared' }
+export interface PreregDiagnosticRun { ok: boolean; command: string; fixture: string; fixture_sha256: string | null; exit: number | null; rows: DiagnosticRow[]; error: string | null }
+
+/**
+ * The optional `## Diagnostics` section: `summarizer: <command naming {fixture}>`,
+ * `fixture: <repo-relative path>` and a `| diagnostic | expected |` table with one
+ * row per preregistered diagnostic. Absent or empty: nothing declared (null).
+ */
+export function parsePreregDiagnostics(text: string): PreregDiagnostics | null {
+  const body = sectionBody(text, 'Diagnostics');
+  if (!body?.length) return null;
+  const d: PreregDiagnostics = { summarizer: null, fixture: null, expected: [], errors: [] };
+  const value = (raw: string) => raw.trim().replace(/^`(.+)`$/, '$1').trim();
+  for (const line of body) {
+    const kv = /^\s*(?:[-*]\s+)?(summarizer|fixture):\s*(.+)$/i.exec(line);
+    if (kv) { d[kv[1]!.toLowerCase() as 'summarizer' | 'fixture'] = value(kv[2]!); continue; }
+    const row = /^\s*\|(.+)\|\s*$/.exec(line);
+    if (!row || /^[\s:|-]+$/.test(row[1]!)) continue;
+    const cells = row[1]!.split('|').map(value);
+    if (cells[0]!.toLowerCase() === 'diagnostic') continue;
+    if (cells.length < 2 || !cells[0] || !/^\d+$/.test(cells[1]!)) { d.errors.push(`row ${JSON.stringify(line.trim())}: needs | <diagnostic> | <expected count, a non-negative integer> |`); continue; }
+    if (d.expected.some(e => e.name === cells[0])) { d.errors.push(`diagnostic ${cells[0]} is declared twice`); continue; }
+    d.expected.push({ name: cells[0]!, count: Number(cells[1]) });
+  }
+  if (!d.summarizer) d.errors.push('no summarizer: line (the command that summarizes the run records)');
+  else if (!d.summarizer.includes(PREREG_FIXTURE_PLACEHOLDER)) d.errors.push(`the summarizer command never names the fixture (${PREREG_FIXTURE_PLACEHOLDER})`);
+  if (!d.fixture) d.errors.push('no fixture: line (the synthetic records file the summarizer runs on)');
+  if (!d.expected.length) d.errors.push('no | diagnostic | expected | rows');
+  return d;
+}
+
+/**
+ * Run the declared summarizer on the declared fixture (a repo command: the
+ * caller gates it on --allow-repo-commands). Its last non-empty stdout line is
+ * one JSON object of diagnostic name to count; every declared diagnostic must
+ * appear with its expected count and nothing undeclared may appear.
+ */
+export function runPreregDiagnostics(d: PreregDiagnostics, repoRoot: string, env: NodeJS.ProcessEnv = process.env): PreregDiagnosticRun {
+  const command = d.summarizer!.split(PREREG_FIXTURE_PLACEHOLDER).join('"$GSTACK_PREREG_FIXTURE"');
+  const fixture = path.resolve(repoRoot, d.fixture!);
+  const run: PreregDiagnosticRun = { ok: false, command, fixture: d.fixture!, fixture_sha256: null, exit: null, rows: [], error: null };
+  if (!fs.existsSync(fixture) || !fs.statSync(fixture).isFile()) return { ...run, error: `declared fixture ${d.fixture} is not a file under ${repoRoot}` };
+  run.fixture_sha256 = createHash('sha256').update(fs.readFileSync(fixture)).digest('hex');
+  const r = spawnSync('sh', ['-c', command], { cwd: repoRoot, env: { ...env, GSTACK_PREREG_FIXTURE: fixture }, encoding: 'utf8', timeout: PREREG_SUMMARIZER_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024 });
+  run.exit = r.status;
+  if (r.status === null) return { ...run, error: r.signal ? `summarizer did not finish within ${PREREG_SUMMARIZER_TIMEOUT_MS / 1000} s (${r.signal})` : `summarizer did not start: ${r.error?.message ?? 'unknown error'}` };
+  const tail = (r.stderr ?? '').trim().split('\n').slice(-1)[0] ?? '';
+  if (r.status !== 0) return { ...run, error: `summarizer exited ${r.status}${tail ? `: ${tail}` : ''}` };
+  const last = (r.stdout ?? '').trim().split('\n').filter(l => l.trim()).slice(-1)[0] ?? '';
+  let counts: Record<string, unknown> | null = null;
+  try { const v = JSON.parse(last); counts = v && typeof v === 'object' && !Array.isArray(v) ? v : null; } catch { counts = null; }
+  if (!counts) return { ...run, error: `the summarizer's last stdout line is not a JSON object of diagnostic counts (${JSON.stringify(last.slice(0, 120))})` };
+  const bad = Object.entries(counts).filter(([, v]) => !Number.isInteger(v) || (v as number) < 0).map(([k]) => k);
+  if (bad.length) return { ...run, error: `the summarizer reported non-count values for ${bad.join(', ')}` };
+  for (const e of d.expected) {
+    const got = counts[e.name] === undefined ? null : counts[e.name] as number;
+    run.rows.push({ name: e.name, expected: e.count, got, status: got === null ? 'missing' : got === e.count ? 'ok' : 'mismatch' });
+  }
+  for (const [name, got] of Object.entries(counts)) if (!d.expected.some(e => e.name === name)) run.rows.push({ name, expected: null, got: got as number, status: 'undeclared' });
+  return { ...run, ok: run.rows.every(row => row.status === 'ok') };
+}
+
+export function renderPreregDiagnostics(run: PreregDiagnosticRun): string {
+  const rows = run.rows.map(r => `PREREG_DIAGNOSTIC: ${r.name} expected=${r.expected ?? 'undeclared'} got=${r.got ?? 'none'} ${r.status}`);
+  rows.push(`PREREG_DIAGNOSTICS: ${run.ok ? 'ok' : 'untested'} n=${run.rows.filter(r => r.status !== 'undeclared').length} fixture=${run.fixture}${run.fixture_sha256 ? ` sha256=${run.fixture_sha256.slice(0, 12)}` : ''}${run.error ? ` error=${JSON.stringify(run.error)}` : ''}`);
+  return rows.join('\n') + '\n';
 }
 
 // ---------------------------------------------------------------------------
@@ -137,10 +219,12 @@ export interface DryRunResult {
     price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null };
     /** Present with `cap`: one call per arm (the largest row's input plus the output budget), priced, fits under the cap. */
     reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record<string, number | null> };
+    /** Present when the caller linted the arms against .gstack/eval-models.json (lib/eval-models.ts). */
+    model_rules?: { ok: boolean; source: string; refused: string[] };
   };
   per_item_usd: Record<string, number | null>; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)';
 }
-export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number }
+export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number; modelRules?: ModelRulesCheck }
 
 /** Four bytes per token, the estimator every gstack admission uses. */
 export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4);
@@ -179,9 +263,10 @@ export function dryRun(o: DryRunOptions): DryRunResult {
     for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null;
     checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm };
   }
+  if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) };
   return {
     schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem,
-    excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true), stub: 'stub-model (no network, $0)',
+    excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true) && (checks.model_rules?.ok ?? true), stub: 'stub-model (no network, $0)',
   };
 }
 
@@ -193,6 +278,7 @@ export function renderDryRun(r: DryRunResult): string {
     `CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`,
     `CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt <file> with the exact text the run will send)'}`,
     ...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []),
+    ...(r.checks.model_rules ? [`CHECK: model-rules ${r.checks.model_rules.ok ? 'ok' : `refused ${r.checks.model_rules.refused.join('; ') || 'invalid rules'}`} (source ${r.checks.model_rules.source})`] : []),
     ...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`),
     `DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`,
   ];

lib/result-codes.ts

@@ -667,6 +667,33 @@ export const RESULT_CODES = {
     summary: '`gstack-autoplan export --public <dir>` writes only into an empty directory or over an earlier public copy of the same run',
     fix: 'pass an empty or new --public directory, or the directory that already holds this run\'s public copy',
   },
+  // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit.
+  PREREG_DIAGNOSTIC_UNTESTED: {
+    anchor: 'prereg-diagnostic-untested',
+    summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count',
+    fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted',
+  },
+  // Field list 2 (2026-10-16), W9: .gstack/eval-models.json.
+  EVAL_MODELS_INVALID: {
+    anchor: 'eval-models-invalid',
+    summary: 'the eval model rules (.gstack/eval-models.json) are unreadable or invalid, so no arm can be checked',
+    fix: 'fix what the line names (docs/eval-models.md: schema_version 1, a non-empty allow, optional deny, scopes and families newest first), or fetch the base branch; no arm passes until the rules parse',
+  },
+  EVAL_MODEL_DENIED: {
+    anchor: 'eval-model-denied',
+    summary: 'an arm names a model the eval model rules deny, or one no allow entry covers for its scope',
+    fix: 'pick a model the rules allow for that scope (the EVAL_MODEL line names the rule), or change .gstack/eval-models.json in a reviewed PR',
+  },
+  EVAL_MODEL_OLD_GENERATION: {
+    anchor: 'eval-model-old-generation',
+    summary: 'an arm names an older generation of a model family the eval model rules order newest first',
+    fix: 'use the newest id of the family the EVAL_MODEL line names, or reorder families in .gstack/eval-models.json in a reviewed PR',
+  },
+  EVAL_MODEL_GENERATION_UNRESOLVED: {
+    anchor: 'eval-model-generation-unresolved',
+    summary: 'the eval model rules declare families (newest generation only) and an arm names a model in none of them, so its generation cannot be checked',
+    fix: 'add the model to its family in .gstack/eval-models.json (newest first) in a reviewed PR, or pick a model a family lists',
+  },
 } as const satisfies Record<string, ResultCode>;
 
 export type ResultCodeName = keyof typeof RESULT_CODES;