diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index accf81c5b..649592135 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -2,7 +2,7 @@ /** * gstack-eval-plan — the tier-1 machinery behind /eval-plan (plan E1). * - * gstack-eval-plan prereg --check | --init [--title ] + * gstack-eval-plan prereg --check [--allow-repo-commands] [--json] | --init [--title ] * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] @@ -29,7 +29,8 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan'; +import { checkPrereg, DEFAULT_TOKENS, renderPreregDiagnostics, runPreregDiagnostics, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan'; +import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; import { resolveBaseBranch } from '../lib/ship-policy'; @@ -37,7 +38,10 @@ import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) - prereg --check the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure + prereg --check [--allow-repo-commands] [--json] + the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure; + an optional Diagnostics section's summarizer runs on its synthetic fixture only with + --allow-repo-commands (else it is listed and exit 3) and every count must match prereg --init [--title ] write the template arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json] arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item @@ -68,6 +72,24 @@ function num(args: string[], name: string, fallback?: number): number | undefine function usage(message: string): number { process.stderr.write(`gstack-eval-plan: ${message}\n${HELP}\n`); return EXIT.usage; } const csv = (v: string | undefined) => (v ?? '').split(',').map(s => s.trim()).filter(Boolean); +function scopeOf(args: string[]): EvalScope { + const v = flag(args, '--scope') ?? 'full'; + if (!(EVAL_SCOPES as readonly string[]).includes(v)) throw new Error(`--scope needs one of ${EVAL_SCOPES.join(', ')}`); + return v as EvalScope; +} + +/** Lint arms against .gstack/eval-models.json (origin/, or the working tree with --models-from head) and print its lines. */ +function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScope; where: string }>, command: string): ModelRulesCheck { + const from = flag(args, '--models-from') ?? 'base'; + if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head'); + const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 }); + const repoRoot = top.status === 0 ? top.stdout.trim() : null; + const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); + if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n'); + for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n'); + return check; +} + function doPrereg(rest: string[]): number { const init = flag(rest, '--init'); if (init) { @@ -80,18 +102,33 @@ function doPrereg(rest: string[]): number { const file = flag(rest, '--check'); if (!file || !fs.existsSync(file)) return usage('prereg needs --check or --init '); const r = checkPrereg(fs.readFileSync(file, 'utf8')); - if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r }) + '\n'); - else process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''} file=${file}\n`); - if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`)].join('; ')) + '\n'); return EXIT.fail; } + const d = r.diagnostics; + const runnable = d && !d.errors.length; + const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: path.dirname(path.resolve(file)), encoding: 'utf8', timeout: 30_000 }); + const repoRoot = top.status === 0 ? top.stdout.trim() : path.dirname(path.resolve(file)); + const allowed = rest.includes('--allow-repo-commands'); + const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null; + if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n'); + else { + process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); + if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`); + if (run) process.stdout.write(renderPreregDiagnostics(run)); + } + if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n'); + if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n'); + if (!r.ok || (run && !run.ok)) return EXIT.fail; + if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; } return EXIT.ok; } function doArms(rest: string[]): number { const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! }; + const scope = scopeOf(rest); const r = resolveArms({ cwd: process.cwd(), explicit: csv(flag(rest, '--arms')), policyDoc: flag(rest, '--policy-doc'), tokens }); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r, tokens }) + '\n'); else process.stdout.write(renderArms(r, tokens)); for (const e of r.errors) process.stderr.write(resultLine('gstack-eval-plan arms', e.code, e.message) + '\n'); - return r.errors.length ? EXIT.fail : EXIT.ok; + const rules = r.arms.length ? lintModels(rest, r.arms.map(a => ({ model: a.model, scope, where: a.source })), 'arms') : null; + return r.errors.length || (rules && !rules.ok) ? EXIT.fail : EXIT.ok; } function doDryRun(rest: string[]): number { @@ -99,13 +136,15 @@ function doDryRun(rest: string[]): number { const arms = csv(flag(rest, '--arms')); if (!input || !fs.existsSync(input)) return usage('dry-run needs --input '); if (!arms.length) return usage('dry-run needs --arms '); - const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap') }); + const scope = scopeOf(rest); + const modelRules = lintModels(rest, arms.map(model => ({ model, scope, where: 'dry-run --arms' })), 'dry-run'); + const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules }); const out = flag(rest, '--out'); if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); } if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r)); if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`); if (!r.pass) { - const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : ''].filter(Boolean).join('; '); + const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : '', r.checks.model_rules && !r.checks.model_rules.ok ? 'model rules refused an arm (EVAL_MODEL lines above)' : ''].filter(Boolean).join('; '); process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n'); return EXIT.fail; } @@ -119,6 +158,8 @@ function doPilot(rest: string[]): number { if (sub === 'start') { const cap = num(args, '--cap'); const arms = csv(flag(args, '--arms')); const dry = flag(args, '--dry-run'); if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap --arms --dry-run '); + const rules = lintModels(args, arms.map(model => ({ model, scope: scopeOf(args), where: 'pilot start --arms' })), 'pilot start'); + if (!rules.ok) return EXIT.fail; const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry }); if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; } for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`); @@ -150,7 +191,7 @@ function doPilot(rest: string[]): number { return usage(`unknown pilot command ${sub ?? ''}`); } -const PRICE_GATE_FLAGS = new Set(['--gate', '--gate-file', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out']); +const PRICE_GATE_FLAGS = new Set(['--gate', '--gate-file', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out', '--models-from']); function doPriceGate(rest: string[]): number { for (let i = 0; i < rest.length; i++) { @@ -171,6 +212,7 @@ function doPriceGate(rest: string[]): number { const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) }); if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; } const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') }); + if (r.receipt?.rows.length && !lintModels(rest, r.receipt.rows.map(row => ({ model: row.model, scope: row.scope, where: `purpose=${row.purpose}` })), 'price-gate').ok) return EXIT.fail; if (r.receipt) { const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate')); const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`); diff --git a/lib/eval-models.ts b/lib/eval-models.ts new file mode 100644 index 000000000..878b8fecf --- /dev/null +++ b/lib/eval-models.ts @@ -0,0 +1,160 @@ +/** + * eval-models — a repo's eval model rules, .gstack/eval-models.json, and the + * lint gstack-eval-plan runs over every arm it prices or reserves (field + * list 2, W9). The file states the rules as data, never prose: + * + * allow ids or globs an arm may use; required, and an id it does not + * match fails closed (an uncatalogued or misspelled model is + * never admitted by default) + * deny ids or globs no arm may use, checked first + * scopes per eval scope (smoke, full: the call-plan row scope; an arm + * with no stated scope is full) an extra allow and/or deny list, + * so "Fable in smoke only" is scopes.full.deny ["claude-fable-*"] + * families optional ordered map, newest first: an arm must be the newest + * id of its family, and with the map present an arm in no family + * fails as unresolved instead of passing unchecked + * + * Literal allow/deny apply whatever lib/model-catalog.ts holds. MODEL_CATALOG + * (one current model per provider and review tier) only checks the families + * map's currency: a catalog model listed after another id in its family is a + * stale map. The reviewed copy is read from origin/; the working tree's + * copy only with --models-from head. Doc: docs/eval-models.md. + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { MODEL_CATALOG } from './model-catalog'; +import type { ResultCodeName } from './result-codes'; + +export const EVAL_MODELS_PATH = '.gstack/eval-models.json'; +export const EVAL_SCOPES = ['smoke', 'full'] as const; +export type EvalScope = typeof EVAL_SCOPES[number]; +export interface ScopeRules { allow?: string[]; deny: string[] } +export interface EvalModelRules { schema_version: 1; allow: string[]; deny: string[]; scopes: Partial>; families: Record } +export interface ArmUse { model: string; scope: EvalScope; where: string } +export type ModelVerdict = 'ok' | 'denied' | 'old-generation' | 'generation-unresolved'; +export interface ModelFinding extends ArmUse { verdict: ModelVerdict; reason: string } +export interface ModelRulesCheck { ok: boolean; source: string; lines: string[]; findings: ModelFinding[]; errors: Array<{ code: ResultCodeName; message: string }> } + +const PATTERN = /^[A-Za-z0-9._:/*?-]+$/; +const TOP_KEYS = ['schema_version', 'allow', 'deny', 'scopes', 'families']; + +function patterns(v: unknown, at: string, errors: string[], o: { required?: boolean; literal?: boolean } = {}): string[] { + if (v === undefined && !o.required) return []; + if (!Array.isArray(v) || (o.required && v.length === 0)) { errors.push(`${at} must be ${o.required ? 'a non-empty' : 'an'} array of model ids${o.literal ? '' : ' or globs'}`); return []; } + const out: string[] = []; + v.forEach((p, i) => { + if (typeof p !== 'string' || !PATTERN.test(p) || (o.literal && /[*?]/.test(p))) errors.push(`${at}[${i}] ${JSON.stringify(p)} is not a model id${o.literal ? ' (family entries are literal ids, newest first)' : ' or glob (letters, digits, . _ : / - and the wildcards * ?)'}`); + else out.push(p); + }); + return out; +} + +const isObject = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v); + +/** Parse and validate the rules; any unknown key, scope or malformed entry fails the whole file. */ +export function parseEvalModels(text: string): { rules: EvalModelRules } | { errors: string[] } { + let doc: unknown; + try { doc = JSON.parse(text); } catch (e: any) { return { errors: [`not valid JSON: ${e.message}`] }; } + if (!isObject(doc)) return { errors: ['must be a JSON object'] }; + const errors: string[] = []; + for (const k of Object.keys(doc)) if (!TOP_KEYS.includes(k)) errors.push(`unknown key ${JSON.stringify(k)} (known: ${TOP_KEYS.join(', ')})`); + if (doc.schema_version !== 1) errors.push('schema_version must be 1'); + const rules: EvalModelRules = { schema_version: 1, allow: patterns(doc.allow, 'allow', errors, { required: true }), deny: patterns(doc.deny, 'deny', errors), scopes: {}, families: {} }; + if (doc.scopes !== undefined && !isObject(doc.scopes)) errors.push('scopes must be an object keyed by eval scope'); + for (const [scope, v] of Object.entries(isObject(doc.scopes) ? doc.scopes : {})) { + if (!(EVAL_SCOPES as readonly string[]).includes(scope)) { errors.push(`scopes.${scope}: unknown eval scope (known: ${EVAL_SCOPES.join(', ')})`); continue; } + if (!isObject(v)) { errors.push(`scopes.${scope} must be an object with allow and/or deny`); continue; } + const extra = Object.keys(v).filter(k => k !== 'allow' && k !== 'deny'); + if (extra.length) errors.push(`scopes.${scope}: unknown key(s) ${extra.join(', ')} (known: allow, deny)`); + rules.scopes[scope as EvalScope] = { ...(v.allow !== undefined ? { allow: patterns(v.allow, `scopes.${scope}.allow`, errors, { required: true }) } : {}), deny: patterns(v.deny, `scopes.${scope}.deny`, errors) }; + } + if (doc.families !== undefined && !isObject(doc.families)) errors.push('families must be an object of family name to model ids, newest first'); + const owner = new Map(); + for (const [family, ids] of Object.entries(isObject(doc.families) ? doc.families : {})) { + rules.families[family] = patterns(ids, `families.${family}`, errors, { required: true, literal: true }); + for (const id of rules.families[family]!) { + if (owner.has(id)) errors.push(`families: ${id} is listed in both ${owner.get(id)} and ${family}`); + else owner.set(id, family); + } + } + for (const entry of MODEL_CATALOG) { + const family = owner.get(entry.model); + const head = family ? rules.families[family]![0] : entry.model; + if (head !== entry.model) errors.push(`families.${family} lists ${head} before ${entry.model}, gstack's current ${entry.provider} ${entry.tier} model (lib/model-catalog.ts, verified ${entry.verifiedAt}); list newest first`); + } + return errors.length ? { errors } : { rules }; +} + +/** The id as given, without a provider prefix, and a Claude id without its date suffix: the forms a rule may name. */ +function forms(model: string): string[] { + const bare = model.replace(/^[a-z-]+[/:]/, ''); + return [...new Set([model, bare, /^claude-/.test(bare) ? bare.replace(/-\d{8}$/, '') : bare])]; +} +const glob = (p: string) => new RegExp(`^${p.split('').map(c => (c === '*' ? '.*' : c === '?' ? '.' : c.replace(/[.+^${}()|[\]\\]/g, '\\$&'))).join('')}$`); +const firstMatch = (list: string[] | undefined, model: string) => (list ?? []).find(p => forms(model).some(f => glob(p).test(f))); + +/** Deny, the scope's deny, allow, the scope's allow, then the newest-generation rule; the first that refuses decides. */ +export function lintArmModels(rules: EvalModelRules, uses: ArmUse[]): ModelFinding[] { + const families = Object.entries(rules.families); + return uses.map((use): ModelFinding => { + const scope = rules.scopes[use.scope]; + const denied = firstMatch(rules.deny, use.model); + if (denied) return { ...use, verdict: 'denied', reason: `deny ${denied}` }; + const scopeDenied = firstMatch(scope?.deny, use.model); + if (scopeDenied) return { ...use, verdict: 'denied', reason: `scopes.${use.scope}.deny ${scopeDenied}` }; + const allowed = firstMatch(rules.allow, use.model); + if (!allowed) return { ...use, verdict: 'denied', reason: 'matches no allow entry (unknown ids fail closed)' }; + if (scope?.allow && !firstMatch(scope.allow, use.model)) return { ...use, verdict: 'denied', reason: `matches no scopes.${use.scope}.allow entry` }; + if (!families.length) return { ...use, verdict: 'ok', reason: `allow ${allowed}` }; + const family = families.find(([, ids]) => ids.some(id => forms(use.model).includes(id))); + if (!family) return { ...use, verdict: 'generation-unresolved', reason: 'in no families entry, so the newest-generation rule cannot be checked' }; + if (!forms(use.model).includes(family[1][0]!)) return { ...use, verdict: 'old-generation', reason: `families.${family[0]} newest is ${family[1][0]}` }; + return { ...use, verdict: 'ok', reason: `allow ${allowed}; newest in families.${family[0]}` }; + }); +} + +const VERDICT_CODE: Record, ResultCodeName> = { denied: 'EVAL_MODEL_DENIED', 'old-generation': 'EVAL_MODEL_OLD_GENERATION', 'generation-unresolved': 'EVAL_MODEL_GENERATION_UNRESOLVED' }; + +function gitText(repoRoot: string, args: string[]): string | null { + const r = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8', timeout: 30_000 }); + return r.status === 0 ? r.stdout : null; +} + +export interface EvalModelsLoad { text: string | null; label: string; notes: string[]; error: string | null } + +/** The reviewed rules from origin/ (default) or the working tree's copy; text null when none are declared. */ +export function loadEvalModels(repoRoot: string | null, o: { from: 'base' | 'head'; base: string }): EvalModelsLoad { + if (!repoRoot) return { text: null, label: 'none (not a git repository)', notes: [], error: null }; + const headFile = path.join(repoRoot, EVAL_MODELS_PATH); + const head = fs.existsSync(headFile) ? fs.readFileSync(headFile, 'utf8') : null; + if (o.from === 'head') return { text: head, label: head === null ? `none (no ${EVAL_MODELS_PATH} in the working tree)` : `head (unreviewed) ${EVAL_MODELS_PATH}`, notes: [], error: null }; + const ref = `origin/${o.base.replace(/^origin\//, '')}`; + const sha = gitText(repoRoot, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`])?.trim(); + if (!sha) { + if (head === null) return { text: null, label: `none (${ref} is not a resolvable commit)`, notes: [], error: null }; + return { text: null, label: ref, notes: [], error: `${ref} is not a resolvable commit, so the reviewed ${EVAL_MODELS_PATH} cannot be read; fetch it, or pass --models-from head to lint with the working-tree copy` }; + } + const at = `${ref}@${sha.slice(0, 12)}`; + const base = gitText(repoRoot, ['show', `${sha}:${EVAL_MODELS_PATH}`]); + if (base === null) return { text: null, label: `none at ${at}`, notes: head === null ? [] : [`the working tree has an unreviewed ${EVAL_MODELS_PATH}; --models-from head lints with it`], error: null }; + return { text: base, label: `${at}:${EVAL_MODELS_PATH}`, notes: head !== null && head !== base ? [`the working-tree ${EVAL_MODELS_PATH} differs from the reviewed copy; linting with ${ref}`] : [], error: null }; +} + +/** Load, validate and lint: one EVAL_MODEL line per arm, then the EVAL_MODELS summary; a refusal or an invalid file is an error. */ +export function checkModelRules(repoRoot: string | null, uses: ArmUse[], o: { from: 'base' | 'head'; base: string }): ModelRulesCheck { + const load = loadEvalModels(repoRoot, o); + const notes = load.notes.map(n => `EVAL_MODELS_NOTE: ${n}`); + const invalid = (message: string): ModelRulesCheck => ({ ok: false, source: load.label, lines: [...notes, `EVAL_MODELS: invalid source=${load.label}`], findings: [], errors: [{ code: 'EVAL_MODELS_INVALID', message }] }); + if (load.error) return invalid(load.error); + if (load.text === null) return { ok: true, source: load.label, lines: [...notes, `EVAL_MODELS: ${load.label}`], findings: [], errors: [] }; + const parsed = parseEvalModels(load.text); + if ('errors' in parsed) return invalid(`${load.label}: ${parsed.errors.join('; ')}`); + const findings = lintArmModels(parsed.rules, uses); + const bad = findings.filter(f => f.verdict !== 'ok'); + return { + ok: bad.length === 0, source: load.label, findings, + lines: [...notes, ...findings.map(f => `EVAL_MODEL: ${f.model} scope=${f.scope} ${f.verdict} (${f.reason}) where=${f.where}`), `EVAL_MODELS: ${bad.length ? `fail refused=${bad.length}/${findings.length}` : `ok n=${findings.length}`} source=${load.label}`], + errors: bad.map(f => ({ code: VERDICT_CODE[f.verdict as Exclude], message: `${f.model} (scope ${f.scope}, ${f.where}): ${f.reason}` })), + }; +} diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 3cfd12eba..33542c5fa 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -17,13 +17,14 @@ import { modelsMain } from './model-policy-cli'; import { isConfigUnavailable, loadConfigFromBase } from './pregate/config'; import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pricingRow } from './pricing'; import type { ResultCodeName } from './result-codes'; +import type { ModelRulesCheck } from './eval-models'; import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger'; // --------------------------------------------------------------------------- // preregistration // --------------------------------------------------------------------------- export const PREREG_SECTIONS = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; -export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[] } +export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; diagnostics: PreregDiagnostics | null } export function preregTemplate(title = 'Eval preregistration'): string { return `# ${title} @@ -42,22 +43,103 @@ export function preregTemplate(title = 'Eval preregistration'): string { ## Held-out exposure + `; } -export function checkPrereg(text: string): PreregCheck { +/** A section's content lines (blank lines and one-line `` guidance dropped); null when the heading is absent. */ +function sectionBody(text: string, name: string): string[] | null { const lines = text.split('\n'); const headings = lines.map((l, i) => ({ i, m: /^#{1,6}\s+(.+?)\s*$/.exec(l) })).filter(x => x.m) as Array<{ i: number; m: RegExpExecArray }>; + const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, '')); + if (idx < 0) return null; + const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length; + return lines.slice(headings[idx]!.i + 1, end).filter(l => l.trim() && !/^\s*\s*$/.test(l)); +} + +export function checkPrereg(text: string): PreregCheck { const present: string[] = []; const missing: string[] = []; const empty: string[] = []; for (const name of PREREG_SECTIONS) { - const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, '')); - if (idx < 0) { missing.push(name); continue; } - const start = headings[idx]!.i + 1; - const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length; - const body = lines.slice(start, end).filter(l => l.trim() && !/^\s*\s*$/.test(l)); - if (body.length === 0) empty.push(name); else present.push(name); + const body = sectionBody(text, name); + if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name); } - return { ok: missing.length === 0 && empty.length === 0, present, missing, empty }; + const diagnostics = parsePreregDiagnostics(text); + return { ok: missing.length === 0 && empty.length === 0 && !diagnostics?.errors.length, present, missing, empty, diagnostics }; +} + +// --------------------------------------------------------------------------- +// preregistered diagnostics: the declared summarizer on a declared synthetic fixture +// --------------------------------------------------------------------------- +/** Placeholder the summarizer command names the fixture by; it runs as "$GSTACK_PREREG_FIXTURE", never spliced into the shell text. */ +export const PREREG_FIXTURE_PLACEHOLDER = '{fixture}'; +export const PREREG_SUMMARIZER_TIMEOUT_MS = 300_000; +export interface PreregDiagnostics { summarizer: string | null; fixture: string | null; expected: Array<{ name: string; count: number }>; errors: string[] } +export interface DiagnosticRow { name: string; expected: number | null; got: number | null; status: 'ok' | 'mismatch' | 'missing' | 'undeclared' } +export interface PreregDiagnosticRun { ok: boolean; command: string; fixture: string; fixture_sha256: string | null; exit: number | null; rows: DiagnosticRow[]; error: string | null } + +/** + * The optional `## Diagnostics` section: `summarizer: `, + * `fixture: ` and a `| diagnostic | expected |` table with one + * row per preregistered diagnostic. Absent or empty: nothing declared (null). + */ +export function parsePreregDiagnostics(text: string): PreregDiagnostics | null { + const body = sectionBody(text, 'Diagnostics'); + if (!body?.length) return null; + const d: PreregDiagnostics = { summarizer: null, fixture: null, expected: [], errors: [] }; + const value = (raw: string) => raw.trim().replace(/^`(.+)`$/, '$1').trim(); + for (const line of body) { + const kv = /^\s*(?:[-*]\s+)?(summarizer|fixture):\s*(.+)$/i.exec(line); + if (kv) { d[kv[1]!.toLowerCase() as 'summarizer' | 'fixture'] = value(kv[2]!); continue; } + const row = /^\s*\|(.+)\|\s*$/.exec(line); + if (!row || /^[\s:|-]+$/.test(row[1]!)) continue; + const cells = row[1]!.split('|').map(value); + if (cells[0]!.toLowerCase() === 'diagnostic') continue; + if (cells.length < 2 || !cells[0] || !/^\d+$/.test(cells[1]!)) { d.errors.push(`row ${JSON.stringify(line.trim())}: needs | | |`); continue; } + if (d.expected.some(e => e.name === cells[0])) { d.errors.push(`diagnostic ${cells[0]} is declared twice`); continue; } + d.expected.push({ name: cells[0]!, count: Number(cells[1]) }); + } + if (!d.summarizer) d.errors.push('no summarizer: line (the command that summarizes the run records)'); + else if (!d.summarizer.includes(PREREG_FIXTURE_PLACEHOLDER)) d.errors.push(`the summarizer command never names the fixture (${PREREG_FIXTURE_PLACEHOLDER})`); + if (!d.fixture) d.errors.push('no fixture: line (the synthetic records file the summarizer runs on)'); + if (!d.expected.length) d.errors.push('no | diagnostic | expected | rows'); + return d; +} + +/** + * Run the declared summarizer on the declared fixture (a repo command: the + * caller gates it on --allow-repo-commands). Its last non-empty stdout line is + * one JSON object of diagnostic name to count; every declared diagnostic must + * appear with its expected count and nothing undeclared may appear. + */ +export function runPreregDiagnostics(d: PreregDiagnostics, repoRoot: string, env: NodeJS.ProcessEnv = process.env): PreregDiagnosticRun { + const command = d.summarizer!.split(PREREG_FIXTURE_PLACEHOLDER).join('"$GSTACK_PREREG_FIXTURE"'); + const fixture = path.resolve(repoRoot, d.fixture!); + const run: PreregDiagnosticRun = { ok: false, command, fixture: d.fixture!, fixture_sha256: null, exit: null, rows: [], error: null }; + if (!fs.existsSync(fixture) || !fs.statSync(fixture).isFile()) return { ...run, error: `declared fixture ${d.fixture} is not a file under ${repoRoot}` }; + run.fixture_sha256 = createHash('sha256').update(fs.readFileSync(fixture)).digest('hex'); + const r = spawnSync('sh', ['-c', command], { cwd: repoRoot, env: { ...env, GSTACK_PREREG_FIXTURE: fixture }, encoding: 'utf8', timeout: PREREG_SUMMARIZER_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024 }); + run.exit = r.status; + if (r.status === null) return { ...run, error: r.signal ? `summarizer did not finish within ${PREREG_SUMMARIZER_TIMEOUT_MS / 1000} s (${r.signal})` : `summarizer did not start: ${r.error?.message ?? 'unknown error'}` }; + const tail = (r.stderr ?? '').trim().split('\n').slice(-1)[0] ?? ''; + if (r.status !== 0) return { ...run, error: `summarizer exited ${r.status}${tail ? `: ${tail}` : ''}` }; + const last = (r.stdout ?? '').trim().split('\n').filter(l => l.trim()).slice(-1)[0] ?? ''; + let counts: Record | null = null; + try { const v = JSON.parse(last); counts = v && typeof v === 'object' && !Array.isArray(v) ? v : null; } catch { counts = null; } + if (!counts) return { ...run, error: `the summarizer's last stdout line is not a JSON object of diagnostic counts (${JSON.stringify(last.slice(0, 120))})` }; + const bad = Object.entries(counts).filter(([, v]) => !Number.isInteger(v) || (v as number) < 0).map(([k]) => k); + if (bad.length) return { ...run, error: `the summarizer reported non-count values for ${bad.join(', ')}` }; + for (const e of d.expected) { + const got = counts[e.name] === undefined ? null : counts[e.name] as number; + run.rows.push({ name: e.name, expected: e.count, got, status: got === null ? 'missing' : got === e.count ? 'ok' : 'mismatch' }); + } + for (const [name, got] of Object.entries(counts)) if (!d.expected.some(e => e.name === name)) run.rows.push({ name, expected: null, got: got as number, status: 'undeclared' }); + return { ...run, ok: run.rows.every(row => row.status === 'ok') }; +} + +export function renderPreregDiagnostics(run: PreregDiagnosticRun): string { + const rows = run.rows.map(r => `PREREG_DIAGNOSTIC: ${r.name} expected=${r.expected ?? 'undeclared'} got=${r.got ?? 'none'} ${r.status}`); + rows.push(`PREREG_DIAGNOSTICS: ${run.ok ? 'ok' : 'untested'} n=${run.rows.filter(r => r.status !== 'undeclared').length} fixture=${run.fixture}${run.fixture_sha256 ? ` sha256=${run.fixture_sha256.slice(0, 12)}` : ''}${run.error ? ` error=${JSON.stringify(run.error)}` : ''}`); + return rows.join('\n') + '\n'; } // --------------------------------------------------------------------------- @@ -137,10 +219,12 @@ export interface DryRunResult { price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null }; /** Present with `cap`: one call per arm (the largest row's input plus the output budget), priced, fits under the cap. */ reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record }; + /** Present when the caller linted the arms against .gstack/eval-models.json (lib/eval-models.ts). */ + model_rules?: { ok: boolean; source: string; refused: string[] }; }; per_item_usd: Record; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)'; } -export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number } +export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number; modelRules?: ModelRulesCheck } /** Four bytes per token, the estimator every gstack admission uses. */ export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4); @@ -179,9 +263,10 @@ export function dryRun(o: DryRunOptions): DryRunResult { for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null; checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm }; } + if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) }; return { schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem, - excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true), stub: 'stub-model (no network, $0)', + excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true) && (checks.model_rules?.ok ?? true), stub: 'stub-model (no network, $0)', }; } @@ -193,6 +278,7 @@ export function renderDryRun(r: DryRunResult): string { `CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`, `CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt with the exact text the run will send)'}`, ...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []), + ...(r.checks.model_rules ? [`CHECK: model-rules ${r.checks.model_rules.ok ? 'ok' : `refused ${r.checks.model_rules.refused.join('; ') || 'invalid rules'}`} (source ${r.checks.model_rules.source})`] : []), ...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`), `DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`, ]; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index 556ea280e..5c22d169e 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -667,6 +667,33 @@ export const RESULT_CODES = { summary: '`gstack-autoplan export --public ` writes only into an empty directory or over an earlier public copy of the same run', fix: 'pass an empty or new --public directory, or the directory that already holds this run\'s public copy', }, + // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. + PREREG_DIAGNOSTIC_UNTESTED: { + anchor: 'prereg-diagnostic-untested', + summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count', + fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted', + }, + // Field list 2 (2026-10-16), W9: .gstack/eval-models.json. + EVAL_MODELS_INVALID: { + anchor: 'eval-models-invalid', + summary: 'the eval model rules (.gstack/eval-models.json) are unreadable or invalid, so no arm can be checked', + fix: 'fix what the line names (docs/eval-models.md: schema_version 1, a non-empty allow, optional deny, scopes and families newest first), or fetch the base branch; no arm passes until the rules parse', + }, + EVAL_MODEL_DENIED: { + anchor: 'eval-model-denied', + summary: 'an arm names a model the eval model rules deny, or one no allow entry covers for its scope', + fix: 'pick a model the rules allow for that scope (the EVAL_MODEL line names the rule), or change .gstack/eval-models.json in a reviewed PR', + }, + EVAL_MODEL_OLD_GENERATION: { + anchor: 'eval-model-old-generation', + summary: 'an arm names an older generation of a model family the eval model rules order newest first', + fix: 'use the newest id of the family the EVAL_MODEL line names, or reorder families in .gstack/eval-models.json in a reviewed PR', + }, + EVAL_MODEL_GENERATION_UNRESOLVED: { + anchor: 'eval-model-generation-unresolved', + summary: 'the eval model rules declare families (newest generation only) and an arm names a model in none of them, so its generation cannot be checked', + fix: 'add the model to its family in .gstack/eval-models.json (newest first) in a reviewed PR, or pick a model a family lists', + }, } as const satisfies Record; export type ResultCodeName = keyof typeof RESULT_CODES;