#!/usr/bin/env -S bun --no-env-file --config=/dev/null /** * gstack-eval-plan — the tier-1 machinery behind /eval-plan (plan E1). * * gstack-eval-plan prereg --check [--allow-repo-commands] [--json] | --init [--title ] * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] * gstack-eval-plan price-gate --gate-file | --gate "" [--plan-file ] [--base ] [--cap-usd N] * [--smoke-cap-usd N] [--allow-unpriced ""] [--out ] [--json] * * The dry run spends nothing (stub model) and prints the checklist: price-table * entry per arm, output budget sized for reasoning models, the retry prompt's * exact text. The pilot reserves through lib/spend-ledger.ts (estimated admission, * lock-shared refusal), settles actual cost, and `report` prints the spend table * (spent, reserved, unknown charges, worst-case overrun) and the extrapolation * with its error band before the approval ask. `price-gate` prices a paid gate * from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the * base branch's .gstack/pregate.json read from the HEAD commit) after checking * its binding (exact gate command, head, input hashes); it executes nothing and * writes a receipt (docs/eval-calls.md). arms, dry-run, pilot start and * price-gate lint every arm's model against the base branch's * .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete * prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused. * Logic: lib/eval-plan.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; import { resolveBaseBranch } from '../lib/ship-policy'; import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) prereg --check [--allow-repo-commands] [--json] the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure; an optional Diagnostics section's summarizer runs on its synthetic fixture only with --allow-repo-commands (else it is listed and exit 3) and every count must match prereg --init [--title ] write the template arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json] arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json] stub model over the real rows; checklist: price table, output budget, retry prompt text, and with --cap reservation-fits-cap (one call: largest row + output budget, priced) pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md Model rules: arms, dry-run, pilot start and price-gate check every arm against .gstack/eval-models.json from origin/ (--base b; --models-from head for the unreviewed working-tree copy) at --scope smoke|full (default full; price-gate uses each call row's scope); docs/eval-models.md ${exitTableHelp()}`; function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; } function num(args: string[], name: string, fallback?: number): number | undefined { const v = flag(args, name); if (v === undefined) return fallback; const n = Number(v); if (!Number.isFinite(n) || n < 0) throw new Error(`${name} needs a non-negative number`); return n; } function usage(message: string): number { process.stderr.write(`gstack-eval-plan: ${message}\n${HELP}\n`); return EXIT.usage; } const csv = (v: string | undefined) => (v ?? '').split(',').map(s => s.trim()).filter(Boolean); function scopeOf(args: string[]): EvalScope { const v = flag(args, '--scope') ?? 'full'; if (!(EVAL_SCOPES as readonly string[]).includes(v)) throw new Error(`--scope needs one of ${EVAL_SCOPES.join(', ')}`); return v as EvalScope; } /** Lint arms against .gstack/eval-models.json (origin/, or the working tree with --models-from head) and print its lines. */ function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScope; where: string }>, command: string): ModelRulesCheck { const from = flag(args, '--models-from') ?? 'base'; if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head'); const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 }); const repoRoot = top.status === 0 ? top.stdout.trim() : null; const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n'); for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n'); return check; } function doPrereg(rest: string[]): number { const init = flag(rest, '--init'); if (init) { if (fs.existsSync(init) && !rest.includes('--force')) { process.stderr.write(`gstack-eval-plan: ${init} exists (pass --force)\n`); return EXIT.refused; } fs.mkdirSync(path.dirname(path.resolve(init)), { recursive: true }); fs.writeFileSync(init, preregTemplate(flag(rest, '--title'))); process.stdout.write(`PREREG_INIT: ${init}\n`); return EXIT.ok; } const file = flag(rest, '--check'); if (!file || !fs.existsSync(file)) return usage('prereg needs --check or --init '); const r = checkPrereg(fs.readFileSync(file, 'utf8')); const d = r.diagnostics; const runnable = d && !d.errors.length; const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: path.dirname(path.resolve(file)), encoding: 'utf8', timeout: 30_000 }); const repoRoot = top.status === 0 ? top.stdout.trim() : path.dirname(path.resolve(file)); const allowed = rest.includes('--allow-repo-commands'); const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null; if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n'); else { process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`); if (run) process.stdout.write(renderPreregDiagnostics(run)); } if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n'); if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n'); if (!r.ok || (run && !run.ok)) return EXIT.fail; if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; } return EXIT.ok; } function doArms(rest: string[]): number { const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! }; const scope = scopeOf(rest); const r = resolveArms({ cwd: process.cwd(), explicit: csv(flag(rest, '--arms')), policyDoc: flag(rest, '--policy-doc'), tokens }); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r, tokens }) + '\n'); else process.stdout.write(renderArms(r, tokens)); for (const e of r.errors) process.stderr.write(resultLine('gstack-eval-plan arms', e.code, e.message) + '\n'); const rules = r.arms.length ? lintModels(rest, r.arms.map(a => ({ model: a.model, scope, where: a.source })), 'arms') : null; return r.errors.length || (rules && !rules.ok) ? EXIT.fail : EXIT.ok; } function doDryRun(rest: string[]): number { const input = flag(rest, '--input'); const arms = csv(flag(rest, '--arms')); if (!input || !fs.existsSync(input)) return usage('dry-run needs --input '); if (!arms.length) return usage('dry-run needs --arms '); const scope = scopeOf(rest); const modelRules = lintModels(rest, arms.map(model => ({ model, scope, where: 'dry-run --arms' })), 'dry-run'); const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap'), modelRules }); const out = flag(rest, '--out'); if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); } if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r)); if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`); if (!r.pass) { const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : '', r.checks.model_rules && !r.checks.model_rules.ok ? 'model rules refused an arm (EVAL_MODEL lines above)' : ''].filter(Boolean).join('; '); process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n'); return EXIT.fail; } return EXIT.ok; } function doPilot(rest: string[]): number { const [sub, ...args] = rest; const out = flag(args, '--out'); if (!out) return usage(`pilot ${sub ?? ''} needs --out `); if (sub === 'start') { const cap = num(args, '--cap'); const arms = csv(flag(args, '--arms')); const dry = flag(args, '--dry-run'); if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap --arms --dry-run '); const rules = lintModels(args, arms.map(model => ({ model, scope: scopeOf(args), where: 'pilot start --arms' })), 'pilot start'); if (!rules.ok) return EXIT.fail; const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry }); if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; } for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`); process.stdout.write(`PILOT: status=${r.record.status} reserved=${r.record.attempts.length} cap=$${cap.toFixed(2)} ledger=${path.join(out, 'spend.json')}\n`); if (r.error) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.fail; } return EXIT.ok; } if (sub === 'settle' || sub === 'release') { const attempt = flag(args, '--attempt'); if (!attempt) return usage(`pilot ${sub} needs --attempt `); if (sub === 'release') { const l = releasePilot(out, attempt); process.stdout.write(`RELEASED: ${attempt} reserved=${l.reserved.length}\n`); return EXIT.ok; } const usd = flag(args, '--usd'); if (usd === undefined) return usage('pilot settle needs --usd '); const value = usd === 'unknown' ? 'unknown' as const : Number(usd); if (value !== 'unknown' && (!Number.isFinite(value) || value < 0)) return usage('--usd needs a non-negative number or unknown'); const l = settlePilot(out, attempt, value); process.stdout.write(`SETTLED: ${attempt} usd=${usd} spent=$${l.spent_usd.toFixed(4)} open=${l.reserved.length}\n`); return EXIT.ok; } if (sub === 'report') { const total = num(args, '--total'); if (total === undefined) return usage('pilot report needs --total '); if (!fs.existsSync(path.join(out, 'pilot.json'))) return usage(`${out} has no pilot.json; run pilot start first`); const r = pilotReport(out, { total, workers: num(args, '--workers') }); if (args.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderPilotReport(r)); if (r.record.status === 'budget_exhausted' || r.spend.worst_case_overrun_usd > 0) { process.stderr.write(resultLine('gstack-eval-plan pilot report', 'EVAL_BUDGET_EXHAUSTED', `worst case $${r.spend.worst_case_usd.toFixed(4)} vs cap $${r.record.cap_usd.toFixed(2)}`) + '\n'); return EXIT.fail; } return r.projection.verdict === 'exceeds' ? EXIT.fail : EXIT.ok; } return usage(`unknown pilot command ${sub ?? ''}`); } const PRICE_GATE_FLAGS = new Set(['--gate', '--gate-file', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out', '--models-from']); function doPriceGate(rest: string[]): number { for (let i = 0; i < rest.length; i++) { if (rest[i] === '--json') continue; if (!PRICE_GATE_FLAGS.has(rest[i]!) || rest[i + 1] === undefined) return usage(`price-gate: unknown flag or missing value ${rest[i]}`); i++; } const gateFile = flag(rest, '--gate-file'); if (gateFile && flag(rest, '--gate') !== undefined) return usage('price-gate takes --gate or --gate-file, not both'); if (gateFile && !fs.existsSync(gateFile)) return usage(`--gate-file ${gateFile} does not exist`); const gate = gateFile ? fs.readFileSync(gateFile, 'utf8').replace(/\r?\n$/, '') : flag(rest, '--gate'); if (!gate?.trim()) return usage('price-gate needs the exact gate command: --gate-file (written with your file-write tool) or --gate ""'); const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }); if (top.status !== 0) return usage('price-gate runs inside the git repository whose gate it prices'); const repoRoot = top.stdout.trim(); const planFile = flag(rest, '--plan-file'); if (planFile && !fs.existsSync(planFile)) return usage(`--plan-file ${planFile} does not exist`); const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) }); if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; } const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') }); if (r.receipt?.rows.length && !lintModels(rest, r.receipt.rows.map(row => ({ model: row.model, scope: row.scope, where: `purpose=${row.purpose}` })), 'price-gate').ok) return EXIT.fail; if (r.receipt) { const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate')); const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`); fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(file, JSON.stringify(r.receipt, null, 2) + '\n'); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r.receipt, receipt_path: file }) + '\n'); else process.stdout.write([...r.lines, renderGatePrice(r.receipt, file)].join('\n') + '\n'); } if (r.error) { process.stderr.write(resultLine('gstack-eval-plan price-gate', r.error.code, r.error.message) + '\n'); return EXIT.fail; } return EXIT.ok; } function main(argv: string[]): number { const [cmd, ...rest] = argv; if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } try { if (cmd === 'prereg') return doPrereg(rest); if (cmd === 'arms') return doArms(rest); if (cmd === 'dry-run') return doDryRun(rest); if (cmd === 'pilot') return doPilot(rest); if (cmd === 'price-gate') return doPriceGate(rest); return usage(`unknown command ${cmd}`); } catch (e: any) { if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; } process.stderr.write(`gstack-eval-plan ${cmd}: ${e?.message ?? e}\n`); return e?.message?.includes('needs') ? EXIT.usage : EXIT.fail; } } process.exitCode = main(process.argv.slice(2));