#!/usr/bin/env bun /** * gstack-eval-plan — the tier-1 machinery behind /eval-plan (plan E1). * * gstack-eval-plan prereg --check | --init [--title ] * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] * * The dry run spends nothing (stub model) and prints the checklist: price-table * entry per arm, output budget sized for reasoning models, the retry prompt's * exact text. The pilot reserves through lib/spend-ledger.ts (estimated admission, * lock-shared refusal), settles actual cost, and `report` prints the spend table * (spent, reserved, unknown charges, worst-case overrun) and the extrapolation * with its error band before the approval ask. Exit 0 ok, 1 fail (incomplete * prereg, failed check, exceeded cap), 2 usage, 3 refused. Logic: lib/eval-plan.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, releasePilot, renderArms, renderDryRun, renderPilotReport, resolveArms, settlePilot, startPilot } from '../lib/eval-plan'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) prereg --check the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure prereg --init [--title ] write the template arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json] arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--out dir] [--json] stub model over the real rows; checklist: price table, output budget, retry prompt text pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask ${exitTableHelp()}`; function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; } function num(args: string[], name: string, fallback?: number): number | undefined { const v = flag(args, name); if (v === undefined) return fallback; const n = Number(v); if (!Number.isFinite(n) || n < 0) throw new Error(`${name} needs a non-negative number`); return n; } function usage(message: string): number { process.stderr.write(`gstack-eval-plan: ${message}\n${HELP}\n`); return EXIT.usage; } const csv = (v: string | undefined) => (v ?? '').split(',').map(s => s.trim()).filter(Boolean); function doPrereg(rest: string[]): number { const init = flag(rest, '--init'); if (init) { if (fs.existsSync(init) && !rest.includes('--force')) { process.stderr.write(`gstack-eval-plan: ${init} exists (pass --force)\n`); return EXIT.refused; } fs.mkdirSync(path.dirname(path.resolve(init)), { recursive: true }); fs.writeFileSync(init, preregTemplate(flag(rest, '--title'))); process.stdout.write(`PREREG_INIT: ${init}\n`); return EXIT.ok; } const file = flag(rest, '--check'); if (!file || !fs.existsSync(file)) return usage('prereg needs --check or --init '); const r = checkPrereg(fs.readFileSync(file, 'utf8')); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r }) + '\n'); else process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''} file=${file}\n`); if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`)].join('; ')) + '\n'); return EXIT.fail; } return EXIT.ok; } function doArms(rest: string[]): number { const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! }; const r = resolveArms({ cwd: process.cwd(), explicit: csv(flag(rest, '--arms')), policyDoc: flag(rest, '--policy-doc'), tokens }); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r, tokens }) + '\n'); else process.stdout.write(renderArms(r, tokens)); for (const e of r.errors) process.stderr.write(resultLine('gstack-eval-plan arms', e.code, e.message) + '\n'); return r.errors.length ? EXIT.fail : EXIT.ok; } function doDryRun(rest: string[]): number { const input = flag(rest, '--input'); const arms = csv(flag(rest, '--arms')); if (!input || !fs.existsSync(input)) return usage('dry-run needs --input '); if (!arms.length) return usage('dry-run needs --arms '); const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows') }); const out = flag(rest, '--out'); if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); } if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r)); if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`); if (!r.pass) { const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : ''].filter(Boolean).join('; '); process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n'); return EXIT.fail; } return EXIT.ok; } function doPilot(rest: string[]): number { const [sub, ...args] = rest; const out = flag(args, '--out'); if (!out) return usage(`pilot ${sub ?? ''} needs --out `); if (sub === 'start') { const cap = num(args, '--cap'); const arms = csv(flag(args, '--arms')); const dry = flag(args, '--dry-run'); if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap --arms --dry-run '); const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry }); if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; } for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`); process.stdout.write(`PILOT: status=${r.record.status} reserved=${r.record.attempts.length} cap=$${cap.toFixed(2)} ledger=${path.join(out, 'spend.json')}\n`); if (r.error) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.fail; } return EXIT.ok; } if (sub === 'settle' || sub === 'release') { const attempt = flag(args, '--attempt'); if (!attempt) return usage(`pilot ${sub} needs --attempt `); if (sub === 'release') { const l = releasePilot(out, attempt); process.stdout.write(`RELEASED: ${attempt} reserved=${l.reserved.length}\n`); return EXIT.ok; } const usd = flag(args, '--usd'); if (usd === undefined) return usage('pilot settle needs --usd '); const value = usd === 'unknown' ? 'unknown' as const : Number(usd); if (value !== 'unknown' && (!Number.isFinite(value) || value < 0)) return usage('--usd needs a non-negative number or unknown'); const l = settlePilot(out, attempt, value); process.stdout.write(`SETTLED: ${attempt} usd=${usd} spent=$${l.spent_usd.toFixed(4)} open=${l.reserved.length}\n`); return EXIT.ok; } if (sub === 'report') { const total = num(args, '--total'); if (total === undefined) return usage('pilot report needs --total '); if (!fs.existsSync(path.join(out, 'pilot.json'))) return usage(`${out} has no pilot.json; run pilot start first`); const r = pilotReport(out, { total, workers: num(args, '--workers') }); if (args.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderPilotReport(r)); if (r.record.status === 'budget_exhausted' || r.spend.worst_case_overrun_usd > 0) { process.stderr.write(resultLine('gstack-eval-plan pilot report', 'EVAL_BUDGET_EXHAUSTED', `worst case $${r.spend.worst_case_usd.toFixed(4)} vs cap $${r.record.cap_usd.toFixed(2)}`) + '\n'); return EXIT.fail; } return r.projection.verdict === 'exceeds' ? EXIT.fail : EXIT.ok; } return usage(`unknown pilot command ${sub ?? ''}`); } function main(argv: string[]): number { const [cmd, ...rest] = argv; if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } try { if (cmd === 'prereg') return doPrereg(rest); if (cmd === 'arms') return doArms(rest); if (cmd === 'dry-run') return doDryRun(rest); if (cmd === 'pilot') return doPilot(rest); return usage(`unknown command ${cmd}`); } catch (e: any) { if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; } process.stderr.write(`gstack-eval-plan ${cmd}: ${e?.message ?? e}\n`); return e?.message?.includes('needs') ? EXIT.usage : EXIT.fail; } } process.exitCode = main(process.argv.slice(2));