diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index b52a021fc..b94cc6254 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -168,6 +168,9 @@ gstack_result_anchor() { DRAFT_DIRECTION_UNRESOLVED) printf '%s' 'draft-direction-unresolved' ;; PREGATE_GUARD_FAILED) printf '%s' 'pregate-guard-failed' ;; SEALED_OUTPUT_REFUSED) printf '%s' 'sealed-output-refused' ;; + EVAL_GATE_UNPRICED) printf '%s' 'eval-gate-unpriced' ;; + EVAL_GATE_OVER_CAP) printf '%s' 'eval-gate-over-cap' ;; + EVAL_PLAN_STALE) printf '%s' 'eval-plan-stale' ;; *) return 1 ;; esac } diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 2d5701bf8..718fc6594 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -10,20 +10,29 @@ * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] + * gstack-eval-plan price-gate --gate "" [--plan-file ] [--base ] [--cap-usd N] + * [--smoke-cap-usd N] [--allow-unpriced ""] [--out ] [--json] * * The dry run spends nothing (stub model) and prints the checklist: price-table * entry per arm, output budget sized for reasoning models, the retry prompt's * exact text. The pilot reserves through lib/spend-ledger.ts (estimated admission, * lock-shared refusal), settles actual cost, and `report` prints the spend table * (spent, reserved, unknown charges, worst-case overrun) and the extrapolation - * with its error band before the approval ask. Exit 0 ok, 1 fail (incomplete - * prereg, failed check, exceeded cap), 2 usage, 3 refused. Logic: lib/eval-plan.ts. + * with its error band before the approval ask. `price-gate` prices a paid gate + * from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the + * base branch's .gstack/pregate.json read from the HEAD commit) after checking + * its binding (exact gate command, head, input hashes); it executes nothing and + * writes a receipt (docs/eval-calls.md). Exit 0 ok, 1 fail (incomplete + * prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused. + * Logic: lib/eval-plan.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; -import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, releasePilot, renderArms, renderDryRun, renderPilotReport, resolveArms, settlePilot, startPilot } from '../lib/eval-plan'; +import { spawnSync } from 'node:child_process'; +import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; +import { resolveBaseBranch } from '../lib/ship-policy'; import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) @@ -32,11 +41,16 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri prereg --init [--title ] write the template arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json] arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item - dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--out dir] [--json] - stub model over the real rows; checklist: price table, output budget, retry prompt text + dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json] + stub model over the real rows; checklist: price table, output budget, retry prompt text, + and with --cap reservation-fits-cap (one call: largest row + output budget, priced) pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask + price-gate --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] + price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base + policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a + receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md ${exitTableHelp()}`; @@ -85,13 +99,13 @@ function doDryRun(rest: string[]): number { const arms = csv(flag(rest, '--arms')); if (!input || !fs.existsSync(input)) return usage('dry-run needs --input '); if (!arms.length) return usage('dry-run needs --arms '); - const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows') }); + const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap') }); const out = flag(rest, '--out'); if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); } if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r)); if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`); if (!r.pass) { - const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : ''].filter(Boolean).join('; '); + const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : ''].filter(Boolean).join('; '); process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n'); return EXIT.fail; } @@ -136,6 +150,36 @@ function doPilot(rest: string[]): number { return usage(`unknown pilot command ${sub ?? ''}`); } +const PRICE_GATE_FLAGS = new Set(['--gate', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out']); + +function doPriceGate(rest: string[]): number { + for (let i = 0; i < rest.length; i++) { + if (rest[i] === '--json') continue; + if (!PRICE_GATE_FLAGS.has(rest[i]!) || rest[i + 1] === undefined) return usage(`price-gate: unknown flag or missing value ${rest[i]}`); + i++; + } + const gate = flag(rest, '--gate'); + if (!gate) return usage('price-gate needs --gate ""'); + const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }); + if (top.status !== 0) return usage('price-gate runs inside the git repository whose gate it prices'); + const repoRoot = top.stdout.trim(); + const planFile = flag(rest, '--plan-file'); + if (planFile && !fs.existsSync(planFile)) return usage(`--plan-file ${planFile} does not exist`); + const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) }); + if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; } + const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') }); + if (r.receipt) { + const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate')); + const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`); + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(file, JSON.stringify(r.receipt, null, 2) + '\n'); + if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r.receipt, receipt_path: file }) + '\n'); + else process.stdout.write([...r.lines, renderGatePrice(r.receipt, file)].join('\n') + '\n'); + } + if (r.error) { process.stderr.write(resultLine('gstack-eval-plan price-gate', r.error.code, r.error.message) + '\n'); return EXIT.fail; } + return EXIT.ok; +} + function main(argv: string[]): number { const [cmd, ...rest] = argv; if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } @@ -144,6 +188,7 @@ function main(argv: string[]): number { if (cmd === 'arms') return doArms(rest); if (cmd === 'dry-run') return doDryRun(rest); if (cmd === 'pilot') return doPilot(rest); + if (cmd === 'price-gate') return doPriceGate(rest); return usage(`unknown command ${cmd}`); } catch (e: any) { if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; } diff --git a/docs/eval-calls.md b/docs/eval-calls.md new file mode 100644 index 000000000..69dc9c527 --- /dev/null +++ b/docs/eval-calls.md @@ -0,0 +1,188 @@ +# Eval call plans: pricing a paid gate without running it + +A paid gate (a benchmark, an eval smoke, any command that calls a model +provider for money) is priced before anyone approves it. gstack does not run +the gate, or any other repository command, to learn its cost. The repository +commits a **call-plan file**: plain data that names the exact gate command, the +commit and inputs it was computed from, and one line per call purpose with its +model and token counts. `gstack-eval-plan price-gate` reads that file, checks +that it still describes what is being approved, prices it from +`lib/pricing.ts`, and writes a receipt. Plan: field wave CD2 +(`docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md`, gate decision d5b as amended). + +```bash +gstack-eval-plan price-gate --gate "" --cap-usd 20 [--smoke-cap-usd 2] [--plan-file ] +``` + +## Where the file comes from + +1. `--plan-file `: an operator-given file, read from disk. +2. Otherwise `price_file` in `.gstack/pregate.json` as committed on + `origin/` (`--base`, default origin/HEAD's branch): a repo-relative + path whose bytes are read from the **HEAD commit** (`git show HEAD:`), + never from the working tree. + +Nothing in the file is executed: not the gate command, not a precondition, +not an input path. The file is parsed as text. + +## File format, wire version v1 + +``` +# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed +GSTACK_EVAL_GATE v1: bun run eval:synopsis --questions 50 --arms opus,sol +GSTACK_EVAL_HEAD v1: 0123456789abcdef0123456789abcdef01234567 +GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=evals/synopsis/questions.jsonl +GSTACK_EVAL_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time +GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full +GSTACK_EVAL_CALLS v1: purpose=judge model=gpt-6.1-sol calls=100 in_tokens=300000 out_tokens=50000 max_out_tokens=2048 scope=full +GSTACK_EVAL_CALLS v1 end: lines=2 +``` + +| Line | Count | Meaning | +|---|---|---| +| `GSTACK_EVAL_GATE v1: ` | exactly 1 | the exact gate command this plan prices; compared to `--gate` by string equality | +| `GSTACK_EVAL_HEAD v1: ` | exactly 1 | the full commit sha the plan was computed at | +| `GSTACK_EVAL_INPUT v1: sha256= path=` | 0 or more | each input the counts came from (rows, datasets, prompts), repo-relative, no `..` | +| `GSTACK_EVAL_PRECONDITION: ` | 0 or more | what must hold at run time; every one is echoed as `GATE_PRECONDITION:` | +| `GSTACK_EVAL_CALLS v1: …` | 0 or more | one line per call purpose (unique `purpose` + `model`) | +| `GSTACK_EVAL_CALLS v1 end: lines=` | exactly 1, last | `n` = the number of call lines; a missing or miscounted end line means a partial producer and refuses | + +Blank lines and lines starting with `#` are ignored. Every other line must be +one of the above; an unknown kind, an unknown version (`v2`), a missing or +extra key, a duplicate key, a non-integer count, `calls=0`, or any content +after the end line refuses with `EVAL_DRY_RUN_FAILED`, naming the line. + +### Call line keys + +| Key | Meaning | +|---|---| +| `purpose` | name of the call purpose (`[A-Za-z0-9._:-]+`), for example `answer`, `judge`, `embed` | +| `model` | model id; a provider prefix (`anthropic/`, `openai:`, `voyage:`) is accepted | +| `calls` | number of calls for this purpose (at least 1) | +| `in_tokens` | **aggregate** input tokens across all of this purpose's calls | +| `out_tokens` | **aggregate** expected output tokens across all of this purpose's calls | +| `max_out_tokens` | the **per-call** output reservation (the `max_tokens` the call sends) | +| `scope` | `full` (the counted run) or `smoke` (a small pre-run check) | + +The aggregate totals are what the purpose is expected to cost: +`usd = price(in_tokens, out_tokens)`. The per-call reservation is what one call +can cost at most, which is what a provider or a spend ledger reserves: +`reservation_usd = price(ceil(in_tokens / calls), max_out_tokens)`. + +## Binding checks (`EVAL_PLAN_STALE`) + +The plan refuses, with no receipt, when it no longer describes what is being +approved: + +- the `GSTACK_EVAL_GATE` command differs from `--gate` by even one character; +- the `GSTACK_EVAL_HEAD` commit is not HEAD and any file other than the plan + file itself changed between them (a committed plan file sits one commit on + top of the head it was computed at, so its own commit is allowed); +- an input's sha256 of its bytes in the HEAD commit differs from its + `GSTACK_EVAL_INPUT` line, or the input is not in HEAD; +- the base policy declares a `price_file` that is not in the HEAD commit. + +`--allow-unpriced` never overrides these: only a gate with no plan declared at +all, or a plan with zero call lines, is unpriced. + +## Pricing outcomes + +``` +GATE_PLAN: file=evals/gate.calls source=price_file sha256= head= inputs=1 verified +GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time +GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full +GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full +CHECK: reservation-fits-cap ok (largest one-call reservation $0.1798) +GATE_PRICE: total=6.7 calls=200 purposes=answer,judge receipt=.gstack/tmp/price-gate/price-gate--.json +``` + +| Outcome | Exit | Code | +|---|---|---| +| priced, under the caps | 0 | — | +| a model with no row in `lib/pricing.ts`: that row and the total print `unknown`, never zero | 1 | `EVAL_PRICE_MISSING` | +| total above `--cap-usd` | 1 | `EVAL_GATE_OVER_CAP` | +| one call's reservation above `--cap-usd`, or above `--smoke-cap-usd` for a `scope=smoke` row (`CHECK: reservation-fits-cap exceeds (…)`) | 1 | `EVAL_GATE_OVER_CAP` | +| no plan declared (no `--plan-file` and no `price_file` key) or a file with zero call lines | 1 | `EVAL_GATE_UNPRICED` | +| the same with `--allow-unpriced ""` | 0 | prints `GATE_UNPRICED_OVERRIDE: (owner-authorized override; not authorization to spend)` and records it | +| malformed or partial file | 1 | `EVAL_DRY_RUN_FAILED` | +| stale binding, or a declared `price_file` missing from HEAD (not overridable) | 1 | `EVAL_PLAN_STALE` | +| `origin/` unresolvable while reading `price_file` | 3 | `POLICY_SOURCE_UNAVAILABLE` | +| `.gstack/pregate.json` on the base is invalid | 3 | `PREGATE_CONFIG_INVALID` | +| unknown flag or missing `--gate` | 2 | usage | + +The price is an estimate: the table prices standard text tokens and excludes +the billing categories listed in `lib/pricing.ts` (cache writes, tool fees, +service tiers and long-context surcharges, media tokens, seat-metered runs). + +## Receipt + +Every outcome past the binding checks writes a JSON receipt to `--out` +(default `/.gstack/tmp/price-gate/`), named +`price-gate--.json`, and prints its path on the +`GATE_PRICE:` line. `--json` prints the receipt (plus `receipt_path`) instead +of the lines. + +| Field | Meaning | +|---|---| +| `schema_version` | `1` | +| `kind` | `gstack-eval-price-gate` | +| `created_at`, `gstack_revision` | when, and which gstack (VERSION and commit) priced it | +| `gate_command`, `gate_sha256` | the approved command and its sha256 | +| `head_sha` | HEAD when priced | +| `plan` | `file`, `source` (`flag`, `price_file`, `none`), `sha256` of the plan file bytes, `declared_head` | +| `inputs` | each `{ path, sha256 }`, verified at HEAD | +| `preconditions` | every precondition line | +| `rows` | each call line plus `usd` and `reservation_usd` (`null` when unpriced) | +| `total_usd`, `calls`, `purposes` | the totals; `total_usd` is `null` when unknown | +| `cap_usd`, `smoke_cap_usd` | the caps passed, or `null` | +| `result`, `code` | `priced`, `price_missing`, `over_cap`, `unpriced`, `unpriced_override`, and the result code | +| `unpriced_reason`, `unpriced_override` | why the gate is unpriced, and the owner's override reason | +| `excluded_billing` | the billing categories the estimate leaves out | + +## Reference emitter + +A consuming repo writes the file from the same code that builds its calls, so +the counts cannot drift from the run. `formatEvalCalls` in `lib/eval-plan.ts` +is the reference; copy it: + +```ts +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +const KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const; +type Row = { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' }; + +export function formatEvalCalls(p: { gate: string; head: string; inputs: { path: string; sha256: string }[]; preconditions: string[]; rows: Row[] }): string { + return [ + '# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed', + `GSTACK_EVAL_GATE v1: ${p.gate}`, + `GSTACK_EVAL_HEAD v1: ${p.head}`, + ...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`), + ...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`), + ...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${KEYS.map(k => `${k}=${r[k]}`).join(' ')}`), + `GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`, + ].join('\n') + '\n'; +} + +// Count tokens the way the run will, without calling a provider, then: +const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); +const input = 'evals/synopsis/questions.jsonl'; +const sha256 = createHash('sha256').update(readFileSync(input)).digest('hex'); +// writeFileSync('evals/gate.calls', formatEvalCalls({ gate, head, inputs: [{ path: input, sha256 }], preconditions, rows })); +``` + +Hash the inputs as committed (the file at HEAD), produce the plan at a clean +HEAD, commit the plan file on its own commit, and set +`"price_file": "evals/gate.calls"` in `.gstack/pregate.json`. + +## Related + +- `gstack-eval-plan dry-run --cap ` runs the same reservation-fits-cap + check for an eval's arms: one call (the largest row's input plus the output + budget), priced, must fit under the cap. +- Result codes: [EVAL_GATE_UNPRICED](troubleshooting.md#eval-gate-unpriced), + [EVAL_GATE_OVER_CAP](troubleshooting.md#eval-gate-over-cap), + [EVAL_PLAN_STALE](troubleshooting.md#eval-plan-stale), + [EVAL_PRICE_MISSING](troubleshooting.md#eval-price-missing), + [EVAL_DRY_RUN_FAILED](troubleshooting.md#eval-dry-run-failed). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2ad23a5c9..b7030dce5 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2500,3 +2500,43 @@ it). A reviewed false positive gets one line in the base branch's destination sets `"visibility": "private"` under `sealed` in the base branch's `.gstack/pregate.json`. Capy's pull-request tool may not run local hooks, so run `gstack-pregate sealed` yourself before any push or PR creation. + + +### `EVAL_GATE_UNPRICED` + +**Meaning.** `gstack-eval-plan price-gate` found no call-plan file for the paid +gate: no `--plan-file`, no `price_file` in the base branch's +`.gstack/pregate.json`, or the file has no `GSTACK_EVAL_CALLS` lines. (A +declared `price_file` missing from HEAD is `EVAL_PLAN_STALE`, not this.) A gate whose cost is unknown cannot +be approved as priced. + +**Fix.** The consuming repo commits a call-plan file in the format of +[docs/eval-calls.md](eval-calls.md) and names it in `price_file`. Only when the +owner accepts an unpriced gate, rerun with `--allow-unpriced ""`: it +exits 0, prints `GATE_UNPRICED_OVERRIDE: (owner-authorized override; +not authorization to spend)` and records the reason in the receipt. + + +### `EVAL_GATE_OVER_CAP` + +**Meaning.** The gate's priced total is above `--cap-usd`, or one call's +reservation (its share of the input plus `max_out_tokens`) is above `--cap-usd`, +or above `--smoke-cap-usd` for a `scope=smoke` row. A provider refuses a +reservation the cap cannot hold, so the run would fail after spending. + +**Fix.** Cut questions, arms or `max_out_tokens` in the call plan and +regenerate it, or raise the cap deliberately with the owner. Never split calls +to slip under it. + + +### `EVAL_PLAN_STALE` + +**Meaning.** The call-plan file prices something other than what is being +approved: its `GSTACK_EVAL_GATE` command is not the exact `--gate` string, its +`GSTACK_EVAL_HEAD` is behind HEAD with files other than the plan file changed +since, an input's sha256 at HEAD differs from its `GSTACK_EVAL_INPUT` line, or +the base policy's `price_file` is not in the HEAD commit. `--allow-unpriced` +does not override it. + +**Fix.** Regenerate the plan file at HEAD for the exact gate command (the +repo's own producer), commit it, and rerun `gstack-eval-plan price-gate`. diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index e4a390ff6..402c07414 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -428,6 +428,8 @@ the budget the run will use. The checklist printed must be all `ok`: spends output tokens thinking, and a smaller budget truncates the answer the scorer reads. - `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry with no text is a repeat of the same failure at the same price. +- `CHECK: reservation-fits-cap` (with `--cap `) — one call, the largest row plus the + output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold. - `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt. `DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on @@ -459,8 +461,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. ## Step 4: The approval ask +Price every paid gate the plan names (a benchmark or eval command that spends) from the +repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack): + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "" --cap-usd +``` + +Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`, +`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced; +`--allow-unpriced ""` only on the owner's word. + Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend -table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one +table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one question with the projection as the recommended option only when `verdict=within`: - A) Run `` items at the projected cost (recommended when within the cap) diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 584ce2b45..12f8136d3 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -100,6 +100,8 @@ the budget the run will use. The checklist printed must be all `ok`: spends output tokens thinking, and a smaller budget truncates the answer the scorer reads. - `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry with no text is a repeat of the same failure at the same price. +- `CHECK: reservation-fits-cap` (with `--cap `) — one call, the largest row plus the + output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold. - `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt. `DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on @@ -131,8 +133,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. ## Step 4: The approval ask +Price every paid gate the plan names (a benchmark or eval command that spends) from the +repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack): + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "" --cap-usd +``` + +Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`, +`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced; +`--allow-unpriced ""` only on the owner's word. + Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend -table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one +table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one question with the projection as the recommended option only when `verdict=within`: - A) Run `` items at the projected cost (recommended when within the cap) diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 162bc27e7..3cfd12eba 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -9,9 +9,12 @@ * (lib/pricing.ts), so the spend table always shows `unknown` charges and the * worst-case overrun (one in-flight call per worker) beside `spent`. */ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { modelsMain } from './model-policy-cli'; +import { isConfigUnavailable, loadConfigFromBase } from './pregate/config'; import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pricingRow } from './pricing'; import type { ResultCodeName } from './result-codes'; import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger'; @@ -129,10 +132,15 @@ export const REASONING_OUTPUT_FLOOR = 8_192; export interface DryRunRow { line: number; ok: boolean; missing: string[]; input_tokens: number } export interface DryRunResult { schema_version: 1; input: string; rows: number; bad_rows: DryRunRow[]; est_input_tokens: number; est_output_tokens: number; - arms: string[]; checks: { price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null } }; + arms: string[]; + checks: { + price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null }; + /** Present with `cap`: one call per arm (the largest row's input plus the output budget), priced, fits under the cap. */ + reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record }; + }; per_item_usd: Record; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)'; } -export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number } +export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number } /** Four bytes per token, the estimator every gstack admission uses. */ export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4); @@ -142,7 +150,7 @@ export function dryRun(o: DryRunOptions): DryRunResult { const budget = o.outputBudget ?? REASONING_OUTPUT_FLOOR; const text = fs.readFileSync(o.input, 'utf8'); const lines = text.split('\n'); - const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0; + const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0; let maxInput = 0; lines.forEach((line, i) => { if (!line.trim()) return; if (o.maxRows && rows >= o.maxRows) return; @@ -153,6 +161,7 @@ export function dryRun(o: DryRunOptions): DryRunResult { const missing = fields.filter(f => parsed![f] === undefined || parsed![f] === null || parsed![f] === ''); const t = estimateTokens(fields.map(f => typeof parsed![f] === 'string' ? parsed![f] as string : JSON.stringify(parsed![f] ?? '')).join('\n')); inputTokens += t; + maxInput = Math.max(maxInput, t); if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t }); }); const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null; @@ -164,10 +173,15 @@ export function dryRun(o: DryRunOptions): DryRunResult { price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice }, output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR }, retry_prompt: { ok: !!retryText, text: retryText }, - }; + } as DryRunResult['checks']; + if (o.cap !== undefined) { + const perArm: Record = {}; + for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null; + checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm }; + } return { schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem, - excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok, stub: 'stub-model (no network, $0)', + excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true), stub: 'stub-model (no network, $0)', }; } @@ -178,12 +192,18 @@ export function renderDryRun(r: DryRunResult): string { `CHECK: price-table ${r.checks.price_table.ok ? 'ok' : `missing ${r.checks.price_table.missing.join(',') || '(no arms)'}`}`, `CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`, `CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt with the exact text the run will send)'}`, + ...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []), ...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`), `DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`, ]; return lines.join('\n') + '\n'; } +function renderReservationCheck(c: NonNullable, budget: number): string { + const worst = Math.max(0, ...Object.values(c.per_arm).filter((v): v is number => v !== null)); + return `CHECK: reservation-fits-cap ${c.ok ? 'ok' : 'exceeds'} ($${worst.toFixed(4)} ${c.ok ? '<=' : '>'} cap $${c.cap.toFixed(2)}) one call = ${c.max_input_tokens} in + ${budget} out tokens`; +} + // --------------------------------------------------------------------------- // priced pilot over lib/spend-ledger.ts // --------------------------------------------------------------------------- @@ -275,3 +295,237 @@ export function renderPilotReport(r: PilotReport): string { p.verdict === 'unknown' ? `APPROVAL: not ready — ${r.settled_known < 2 ? 'settle at least 2 pilot attempts with actual cost' : 'unknown charges make the band meaningless'}; settle, then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`, ].join('\n') + '\n'; } + +// --------------------------------------------------------------------------- +// paid gates priced from a call-plan data file (docs/eval-calls.md) +// --------------------------------------------------------------------------- +/** + * The plan file is DATA: `price-gate` reads it, checks its binding (the exact + * gate command, the head it was produced at, each input's sha256 at HEAD) and + * prices its `GSTACK_EVAL_CALLS v1` rows. Nothing in it is ever executed. + */ +export const EVAL_CALL_KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const; +export interface EvalCallRow { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' } +export interface EvalPlanInput { path: string; sha256: string } +export interface EvalCallPlan { gate: string; head: string; inputs: EvalPlanInput[]; preconditions: string[]; rows: EvalCallRow[] } + +/** The reference emitter a consuming repo copies: binding lines, preconditions, one line per call purpose, the end line. */ +export function formatEvalCalls(p: EvalCallPlan): string { + return [ + '# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed', + `GSTACK_EVAL_GATE v1: ${p.gate}`, + `GSTACK_EVAL_HEAD v1: ${p.head}`, + ...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`), + ...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`), + ...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${EVAL_CALL_KEYS.map(k => `${k}=${r[k]}`).join(' ')}`), + `GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`, + ].join('\n') + '\n'; +} + +const INT_KEYS = ['calls', 'in_tokens', 'out_tokens', 'max_out_tokens'] as const; + +function parseCallRow(body: string, at: string): EvalCallRow | string { + const fields: Record = {}; + for (const tok of body.trim().split(/ +/)) { + const eq = tok.indexOf('='); + if (eq <= 0) return `${at}: ${JSON.stringify(tok)} is not key=value`; + const key = tok.slice(0, eq); + if (!(EVAL_CALL_KEYS as readonly string[]).includes(key)) return `${at}: unknown key ${key}`; + if (key in fields) return `${at}: duplicate key ${key}`; + fields[key] = tok.slice(eq + 1); + } + for (const key of EVAL_CALL_KEYS) if (!(key in fields)) return `${at}: missing key ${key}`; + for (const key of INT_KEYS) if (!/^\d+$/.test(fields[key]!)) return `${at}: ${key} must be a non-negative integer (got ${JSON.stringify(fields[key])})`; + if (Number(fields.calls) < 1) return `${at}: calls must be at least 1`; + if (fields.scope !== 'full' && fields.scope !== 'smoke') return `${at}: scope must be full or smoke (got ${JSON.stringify(fields.scope)})`; + if (!/^[A-Za-z0-9._:-]+$/.test(fields.purpose!)) return `${at}: purpose must match [A-Za-z0-9._:-]+`; + if (!fields.model) return `${at}: model is empty`; + return { purpose: fields.purpose!, model: fields.model, calls: Number(fields.calls), in_tokens: Number(fields.in_tokens), out_tokens: Number(fields.out_tokens), max_out_tokens: Number(fields.max_out_tokens), scope: fields.scope }; +} + +/** Strict v1 parser: a malformed line, an unknown kind or version, a missing binding or a missing or miscounted end line refuses, naming the line. */ +export function parseEvalCalls(text: string): { plan: EvalCallPlan } | { error: string } { + const plan: EvalCallPlan = { gate: '', head: '', inputs: [], preconditions: [], rows: [] }; + const seen = new Set(); + let gate: string | null = null; let head: string | null = null; let ended = false; + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]!.replace(/\r$/, ''); const at = `line ${i + 1}`; + if (!raw.trim() || raw.startsWith('#')) continue; + if (ended) return { error: `${at}: content after the end line` }; + const m = /^(GSTACK_EVAL_[A-Z_]+?)(?: (v\d+))?( end)?: (.*)$/.exec(raw); + if (!m) return { error: `${at}: not a GSTACK_EVAL_ line (${JSON.stringify(raw.slice(0, 80))})` }; + const [, kind, version, end, body] = m as unknown as [string, string, string | undefined, string | undefined, string]; + if (kind === 'GSTACK_EVAL_PRECONDITION' && !version && !end) { plan.preconditions.push(body.trim()); continue; } + if (!['GSTACK_EVAL_GATE', 'GSTACK_EVAL_HEAD', 'GSTACK_EVAL_INPUT', 'GSTACK_EVAL_CALLS'].includes(kind)) return { error: `${at}: unknown line kind ${kind}` }; + if (version !== 'v1') return { error: `${at}: ${kind} version ${version ?? '(none)'} is not supported (this gstack reads v1)` }; + if (end) { + const n = kind === 'GSTACK_EVAL_CALLS' ? /^lines=(\d+)$/.exec(body.trim()) : null; + if (!n) return { error: `${at}: malformed end line (expected GSTACK_EVAL_CALLS v1 end: lines=)` }; + if (Number(n[1]) !== plan.rows.length) return { error: `${at}: end line says lines=${n[1]} but the file has ${plan.rows.length} call lines` }; + ended = true; continue; + } + if (kind === 'GSTACK_EVAL_GATE') { + if (gate !== null) return { error: `${at}: second GSTACK_EVAL_GATE line` }; + if (!body.trim()) return { error: `${at}: empty gate command` }; + gate = body; continue; + } + if (kind === 'GSTACK_EVAL_HEAD') { + if (head !== null) return { error: `${at}: second GSTACK_EVAL_HEAD line` }; + if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/.test(body.trim())) return { error: `${at}: head must be a full commit sha` }; + head = body.trim(); continue; + } + if (kind === 'GSTACK_EVAL_INPUT') { + const inp = /^sha256=([0-9a-f]{64}) path=(.+)$/.exec(body); + if (!inp) return { error: `${at}: expected GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=` }; + if (path.isAbsolute(inp[2]!) || inp[2]!.split(/[\\/]/).includes('..')) return { error: `${at}: input path must be repo-relative without .. (${inp[2]})` }; + plan.inputs.push({ path: inp[2]!, sha256: inp[1]! }); continue; + } + const row = parseCallRow(body, at); + if (typeof row === 'string') return { error: row }; + if (seen.has(`${row.purpose}\0${row.model}`)) return { error: `${at}: duplicate purpose=${row.purpose} model=${row.model}` }; + seen.add(`${row.purpose}\0${row.model}`); + plan.rows.push(row); + } + if (gate === null) return { error: 'missing GSTACK_EVAL_GATE v1 line (the exact gate command this plan prices)' }; + if (head === null) return { error: 'missing GSTACK_EVAL_HEAD v1 line (the commit the plan was produced at)' }; + if (!ended) return { error: 'missing end line (GSTACK_EVAL_CALLS v1 end: lines=); a partial file is refused' }; + return { plan: { ...plan, gate, head } }; +} + +export interface PricedCallRow extends EvalCallRow { usd: number | null; reservation_usd: number | null } +export interface GateReceipt { + schema_version: 1; kind: 'gstack-eval-price-gate'; created_at: string; gstack_revision: string; + gate_command: string; gate_sha256: string; head_sha: string; + plan: { file: string | null; source: 'flag' | 'price_file' | 'none'; sha256: string | null; declared_head: string | null }; + inputs: EvalPlanInput[]; preconditions: string[]; rows: PricedCallRow[]; + total_usd: number | null; calls: number; purposes: string[]; cap_usd: number | null; smoke_cap_usd: number | null; + result: 'priced' | 'price_missing' | 'over_cap' | 'unpriced' | 'unpriced_override'; code: ResultCodeName | null; + unpriced_reason: string | null; unpriced_override: string | null; excluded_billing: readonly string[]; +} +export interface PlanSource { text: Buffer | null; file: string | null; source: GateReceipt['plan']['source']; repoPath: string | null; why: string | null } +export interface PriceGateOptions { repoRoot: string; gate: string; plan: PlanSource; capUsd?: number; smokeCapUsd?: number; allowUnpriced?: string; now?: Date } +export interface PriceGateResult { lines: string[]; receipt: GateReceipt | null; error?: { code: ResultCodeName; message: string } } + +const sha256 = (b: Buffer | string) => createHash('sha256').update(b).digest('hex'); +function gitOut(repoRoot: string, args: string[]): Buffer | null { + const r = spawnSync('git', args, { cwd: repoRoot, timeout: 30_000 }); + return r.status === 0 ? r.stdout : null; +} + +export interface PlanSourceRefusal { code: ResultCodeName; message: string; refused: boolean } + +/** + * `--plan-file` wins; else the base policy's `price_file`, read from the HEAD + * commit's bytes. Only "no plan declared" is unpriced (overridable); a declared + * file missing at HEAD is stale, an unresolvable base or an invalid policy refuses. + */ +export function resolvePlanSource(repoRoot: string, o: { planFile?: string; base: string }): PlanSource | PlanSourceRefusal { + if (o.planFile) { + const file = path.resolve(o.planFile); + const rel = path.relative(repoRoot, file); + return { text: fs.readFileSync(file), file, source: 'flag', repoPath: rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel.split(path.sep).join('/') : null, why: null }; + } + const load = loadConfigFromBase(repoRoot, o.base); + if (isConfigUnavailable(load)) return { code: 'POLICY_SOURCE_UNAVAILABLE', message: `no --plan-file and the base policy cannot be read (${load.detail})`, refused: true }; + if (load.errors.length) return { code: 'PREGATE_CONFIG_INVALID', message: `${load.label}: ${load.errors.map(e => `${e.path}: ${e.message}`).join('; ')}`, refused: true }; + const rel = load.config.price_file; + if (!rel) return { text: null, file: null, source: 'none', repoPath: null, why: `no --plan-file and ${load.label} declares no price_file` }; + const text = gitOut(repoRoot, ['show', `HEAD:${rel}`]); + if (!text) return { code: 'EVAL_PLAN_STALE', message: `price_file ${rel} (declared in ${load.label}) is not in the HEAD commit`, refused: false }; + return { text, file: rel, source: 'price_file', repoPath: rel, why: null }; +} + +/** Stale binding: the gate command, the head (only the plan file itself may change after it) and every input hash at HEAD. */ +function checkBinding(o: PriceGateOptions, plan: EvalCallPlan, headSha: string): string | null { + if (plan.gate !== o.gate) return `gate command differs: --gate ${JSON.stringify(o.gate)} vs plan ${JSON.stringify(plan.gate)}`; + if (plan.head !== headSha) { + const diff = gitOut(o.repoRoot, ['diff', '--name-only', plan.head, headSha]); + if (diff === null) return `declared head ${plan.head.slice(0, 12)} is not a commit in this repository (HEAD ${headSha.slice(0, 12)})`; + const changed = diff.toString('utf8').split('\n').filter(Boolean); + if (changed.some(f => f !== o.plan.repoPath)) return `declared head ${plan.head.slice(0, 12)} is behind HEAD ${headSha.slice(0, 12)}: ${changed.length} file(s) changed since (${changed.slice(0, 3).join(', ')})`; + } + for (const input of plan.inputs) { + const bytes = gitOut(o.repoRoot, ['show', `HEAD:${input.path}`]); + if (!bytes) return `input ${input.path} is not in the HEAD commit`; + const got = sha256(bytes); + if (got !== input.sha256) return `input ${input.path} sha256 ${got.slice(0, 12)} at HEAD != declared ${input.sha256.slice(0, 12)}`; + } + return null; +} + +export function gstackRevision(): string { + const root = path.resolve(import.meta.dir, '..'); + const version = fs.existsSync(path.join(root, 'VERSION')) ? fs.readFileSync(path.join(root, 'VERSION'), 'utf8').trim() : 'unknown'; + const sha = gitOut(root, ['rev-parse', '--short', 'HEAD'])?.toString('utf8').trim(); + return sha ? `${version} (${sha})` : version; +} + +const amount = (v: number | null) => (v === null ? 'unknown' : String(+v.toFixed(6))); + +function priceRows(rows: EvalCallRow[]): PricedCallRow[] { + return rows.map(r => { + if (!hasPricing(r.model)) return { ...r, usd: null, reservation_usd: null }; + return { ...r, usd: estimateCostUsd({ input: r.in_tokens, output: r.out_tokens }, r.model), reservation_usd: estimateCostUsd({ input: Math.ceil(r.in_tokens / r.calls), output: r.max_out_tokens }, r.model) }; + }); +} + +/** Price a paid gate from its call-plan file. Malformed or stale files return an error and no receipt; every other outcome returns a receipt. */ +export function priceGate(o: PriceGateOptions): PriceGateResult { + const headSha = gitOut(o.repoRoot, ['rev-parse', 'HEAD'])?.toString('utf8').trim() ?? ''; + let plan: EvalCallPlan | null = null; + if (o.plan.text) { + const parsed = parseEvalCalls(o.plan.text.toString('utf8')); + if ('error' in parsed) return { lines: [], receipt: null, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.plan.file}: ${parsed.error}` } }; + const stale = checkBinding(o, parsed.plan, headSha); + if (stale) return { lines: [], receipt: null, error: { code: 'EVAL_PLAN_STALE', message: `${o.plan.file}: ${stale}` } }; + plan = parsed.plan; + } + const rows = priceRows(plan?.rows ?? []); + const total = rows.some(r => r.usd === null) ? null : +rows.reduce((s, r) => s + r.usd!, 0).toFixed(6); + const lines = [ + ...(plan ? [`GATE_PLAN: file=${o.plan.file} source=${o.plan.source} sha256=${sha256(o.plan.text!)} head=${plan.head.slice(0, 12)}${plan.head === headSha ? '' : ` (HEAD ${headSha.slice(0, 12)}; only the plan file changed since)`} inputs=${plan.inputs.length} verified`] : []), + ...(plan?.preconditions ?? []).map(t => `GATE_PRECONDITION: ${t}`), + ...rows.map(r => `GATE_CALLS: purpose=${r.purpose} model=${r.model} calls=${r.calls} usd=${amount(r.usd)} reservation_usd=${amount(r.reservation_usd)} scope=${r.scope}`), + ]; + let error: PriceGateResult['error']; + let result: GateReceipt['result'] = 'priced'; + const unpricedWhy = !plan ? o.plan.why ?? 'no call-plan file' : rows.length === 0 ? `${o.plan.file} has no GSTACK_EVAL_CALLS lines` : null; + if (unpricedWhy) { + result = o.allowUnpriced ? 'unpriced_override' : 'unpriced'; + if (o.allowUnpriced) lines.push(`GATE_UNPRICED_OVERRIDE: ${o.allowUnpriced} (owner-authorized override; not authorization to spend)`); + else error = { code: 'EVAL_GATE_UNPRICED', message: unpricedWhy }; + } else if (total === null) { + result = 'price_missing'; + error = { code: 'EVAL_PRICE_MISSING', message: `no price row for ${rows.filter(r => r.usd === null).map(r => r.model).join(', ')}; the total is unknown, never zero` }; + } else { + const over: string[] = []; + if (o.capUsd !== undefined || o.smokeCapUsd !== undefined) { + for (const r of rows) { + const smoke = r.scope === 'smoke' && o.smokeCapUsd !== undefined && r.reservation_usd! > o.smokeCapUsd; + const full = o.capUsd !== undefined && r.reservation_usd! > o.capUsd; + if (!smoke && !full) continue; + const why = `$${r.reservation_usd!.toFixed(4)} > ${smoke ? `smoke cap $${o.smokeCapUsd!.toFixed(2)}` : `cap $${o.capUsd!.toFixed(2)}`}`; + lines.push(`CHECK: reservation-fits-cap exceeds (${why}) purpose=${r.purpose}`); + over.push(`reservation-fits-cap: purpose=${r.purpose} one call ${why}`); + } + if (!over.length) lines.push(`CHECK: reservation-fits-cap ok (largest one-call reservation $${Math.max(0, ...rows.map(r => r.reservation_usd!)).toFixed(4)})`); + } + if (o.capUsd !== undefined && total > o.capUsd) over.unshift(`total $${total.toFixed(4)} > cap $${o.capUsd.toFixed(2)}`); + if (over.length) { result = 'over_cap'; error = { code: 'EVAL_GATE_OVER_CAP', message: over.join('; ') }; } + } + const receipt: GateReceipt = { + schema_version: 1, kind: 'gstack-eval-price-gate', created_at: (o.now ?? new Date()).toISOString(), gstack_revision: gstackRevision(), + gate_command: o.gate, gate_sha256: sha256(o.gate), head_sha: headSha, + plan: { file: o.plan.file, source: o.plan.source, sha256: o.plan.text ? sha256(o.plan.text) : null, declared_head: plan?.head ?? null }, + inputs: plan?.inputs ?? [], preconditions: plan?.preconditions ?? [], rows, + total_usd: unpricedWhy ? null : total, calls: rows.reduce((s, r) => s + r.calls, 0), purposes: rows.map(r => r.purpose), + cap_usd: o.capUsd ?? null, smoke_cap_usd: o.smokeCapUsd ?? null, result, code: error?.code ?? (unpricedWhy ? 'EVAL_GATE_UNPRICED' : null), + unpriced_reason: unpricedWhy, unpriced_override: unpricedWhy ? o.allowUnpriced ?? null : null, excluded_billing: EXCLUDED_BILLING_CATEGORIES, + }; + return { lines, receipt, error }; +} + +export function renderGatePrice(r: GateReceipt, receiptPath: string): string { + return `GATE_PRICE: total=${amount(r.total_usd)} calls=${r.calls} purposes=${r.purposes.join(',') || 'none'} receipt=${receiptPath}`; +} diff --git a/lib/pricing.ts b/lib/pricing.ts index 3bf6a0bb2..c3d547399 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -20,7 +20,8 @@ * start a priced run on such a model (bin/gstack-eval-plan dry-run). * - Anthropic: https://www.anthropic.com/pricing#api * - OpenAI: https://openai.com/api/pricing/ - * - Google AI: https://ai.google.dev/pricing + * - Google AI: https://ai.google.dev/gemini-api/docs/pricing + * - Voyage AI (embeddings): https://docs.voyageai.com/docs/pricing */ export interface ModelPricing { @@ -45,7 +46,10 @@ export const PRICING: Record = { 'claude-fable-5-1': { input_per_mtok: 10.00, output_per_mtok: 50.00, cached_input_per_mtok: 0.25, as_of: '2026-09' }, // https://platform.claude.com/docs/en/models/opus-5-5/overview#pricing (list prices effective 2026-10-07) 'claude-opus-5-5': { input_per_mtok: 4.00, output_per_mtok: 20.00, cached_input_per_mtok: 0.20, as_of: '2026-10' }, - 'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' }, + // https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12) + 'claude-opus-4-7': { input_per_mtok: 5.00, output_per_mtok: 25.00, cached_input_per_mtok: 0.50, as_of: '2026-10' }, + // https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12) + 'claude-sonnet-5-5': { input_per_mtok: 2.00, output_per_mtok: 10.00, cached_input_per_mtok: 0.10, as_of: '2026-10' }, 'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' }, 'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' }, @@ -60,14 +64,46 @@ export const PRICING: Record = { 'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' }, // Google + // https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; standard tier through 2026-12-31, + // the page lists $1.50 / $7.50 / $0.15 cached starting 2027-01-01) + 'gemini-3.8-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' }, + // https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; same schedule as gemini-3.8-flash) + 'gemini-3.6-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' }, + // https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; prompts <= 200k tokens, the long-context rate is excluded) + 'gemini-3.1-pro-preview': { input_per_mtok: 2.00, output_per_mtok: 12.00, cached_input_per_mtok: 0.20, as_of: '2026-10' }, 'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' }, 'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' }, }; -/** A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`) resolves to its table row. */ +/** + * Embedding models bill input tokens only (output_per_mtok 0), so they live + * beside PRICING rather than in it: eval arms never resolve to an embedding + * model, but a call plan (docs/eval-calls.md) prices gbrain's embedding calls. + */ +export const EMBEDDING_PRICING: Record = { + // https://docs.voyageai.com/docs/pricing (fetched 2026-10-12; list price after the free-token allowance) + 'voyage-4': { input_per_mtok: 0.06, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-4-large': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-4-lite': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-code-4': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-code-3': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-3-large': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' }, + 'voyage-multimodal-3': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' }, // text tokens; pixels excluded + // https://developers.openai.com/api/docs/pricing (fetched 2026-10-12) + 'text-embedding-3-large': { input_per_mtok: 0.13, output_per_mtok: 0, as_of: '2026-10' }, + 'text-embedding-3-small': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' }, + // https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; text input, standard tier) + 'gemini-embedding-2': { input_per_mtok: 0.20, output_per_mtok: 0, as_of: '2026-10' }, +}; + +/** + * A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`, + * or gbrain's `voyage:voyage-4`) resolves to its table row. + */ export function pricingRow(model: string | undefined): ModelPricing | undefined { if (!model) return undefined; - return PRICING[model] ?? PRICING[model.replace(/^[a-z-]+\//, '')]; + const bare = model.replace(/^[a-z-]+[/:]/, ''); + return PRICING[model] ?? PRICING[bare] ?? EMBEDDING_PRICING[model] ?? EMBEDDING_PRICING[bare]; } export function hasPricing(model: string | undefined): boolean { diff --git a/lib/result-codes.ts b/lib/result-codes.ts index ccc974133..ede87c1ba 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -476,6 +476,21 @@ export const RESULT_CODES = { summary: 'a push or PR would publish sealed or eval output (a sealed path, a transcript, a quoted answer, a seed or a local home path) to a public or unknown destination, or the sealed policy is unreadable', fix: 'remove it from every published commit, or add one reviewed `rulepathshareason` line to the base branch\'s .gstack/sealed-allow.tsv; a private destination sets sealed.visibility "private" in the base branch\'s .gstack/pregate.json', }, + EVAL_GATE_UNPRICED: { + anchor: 'eval-gate-unpriced', + summary: 'the paid gate has no call-plan file (no `--plan-file`, no base-policy `price_file`) or the file has no GSTACK_EVAL_CALLS lines, so its cost is unknown', + fix: 'have the repo commit a call-plan file (docs/eval-calls.md) and set `price_file` in .gstack/pregate.json, or pass `--allow-unpriced ""` only on the owner\'s say-so', + }, + EVAL_GATE_OVER_CAP: { + anchor: 'eval-gate-over-cap', + summary: 'the paid gate\'s priced total, or one call\'s reservation (input plus maximum output), is above the cap', + fix: 'cut questions, arms or `max_out_tokens` in the call plan, or raise `--cap-usd` / `--smoke-cap-usd` deliberately with the owner; never split calls to slip under it', + }, + EVAL_PLAN_STALE: { + anchor: 'eval-plan-stale', + summary: 'the call-plan file\'s binding does not match (a different gate command, a head behind HEAD, an input whose sha256 at HEAD differs) or the declared price_file is not in HEAD', + fix: 'regenerate the call-plan file at HEAD for the exact gate command you are approving, commit it, and rerun `gstack-eval-plan price-gate`', + }, } as const satisfies Record; export type ResultCodeName = keyof typeof RESULT_CODES; diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index 993de4759..f8e627960 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -23,10 +23,10 @@ test('estimateCostUsd returns 0 for unknown model (no crash)', () => { }); test('estimateCostUsd computes correctly for known Claude model', () => { - // claude-opus-4-7: $15/MTok input, $75/MTok output - // 1M input + 0.5M output = $15 + $37.50 = $52.50 + // claude-opus-4-7: $5/MTok input, $25/MTok output (platform.claude.com pricing, fetched 2026-10-12) + // 1M input + 0.5M output = $5 + $12.50 = $17.50 const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7'); - expect(cost).toBeCloseTo(52.50, 2); + expect(cost).toBeCloseTo(17.50, 2); }); for (const model of [CLAUDE_FRONTIER_EVAL_MODEL, CODEX_FRONTIER_MODEL]) { @@ -48,12 +48,12 @@ test('Astra cache reads use its standard cached-input rate', () => { test('estimateCostUsd applies cached input discount alongside uncached input', () => { // tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%. - // 0 uncached input, 1M cached → 10% of 15 = $1.50 + // 0 uncached input, 1M cached → 10% of 5 = $0.50 const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7'); - expect(cost1).toBeCloseTo(1.50, 2); - // 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25 + expect(cost1).toBeCloseTo(0.50, 2); + // 500K uncached input + 500K cached → $2.50 + $0.25 = $2.75 const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7'); - expect(cost2).toBeCloseTo(8.25, 2); + expect(cost2).toBeCloseTo(2.75, 2); }); test('PRICING table covers the key model families', () => { diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index d6e8e413a..a090d05e4 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -3,16 +3,19 @@ * /eval-plan machinery in lib/eval-plan.ts: preregistration check, priced arms, * the $0 dry run, and the priced pilot ledger with its spend table. */ -import { afterEach, describe, expect, test } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { EXCLUDED_BILLING_CATEGORIES, PRICING, estimateCostUsd, hasPricing, pricingRow } from '../lib/pricing'; import * as helperPricing from './helpers/pricing'; import { - PREREG_SECTIONS, checkPrereg, dryRun, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable, + PREREG_SECTIONS, checkPrereg, dryRun, formatEvalCalls, parseEvalCalls, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable, resolveArms, settlePilot, startPilot, } from '../lib/eval-plan'; +import { capyHost, type CapyHost } from './helpers/capy-host'; const dirs: string[] = []; const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-plan-')); dirs.push(d); return d; }; @@ -156,3 +159,273 @@ describe('priced pilot', () => { expect(r.projection.verdict).toBe('unknown'); }); }); + +// --------------------------------------------------------------------------- +// CD2 (field wave item 15): paid gates priced from a call-plan DATA file. +// `price-gate` never executes anything: it validates the file's binding (gate +// command, head, input hashes at HEAD) and prices the GSTACK_EVAL_CALLS v1 rows. +// --------------------------------------------------------------------------- +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin', 'gstack-eval-plan'); + +function scratchRepo(env: Record): { repo: string; head: string; rowsSha: string } { + const repo = path.join(env.TMPDIR!, 'repo'); + fs.mkdirSync(repo, { recursive: true }); + const git = (...args: string[]) => { + const r = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8' }); + if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); + return r.stdout.trim(); + }; + git('init', '-q', '-b', 'main'); + fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n'); + git('add', 'rows.jsonl'); + git('commit', '-q', '-m', 'rows'); + return { repo, head: git('rev-parse', 'HEAD'), rowsSha: createHash('sha256').update(fs.readFileSync(path.join(repo, 'rows.jsonl'))).digest('hex') }; +} + +const GATE = 'bun run eval:synopsis --questions 50 --arms opus,sol'; +const twoRows = [ + { purpose: 'answer', model: 'anthropic/claude-opus-5-5', calls: 100, in_tokens: 400_000, out_tokens: 200_000, max_out_tokens: 8_192, scope: 'full' as const }, + { purpose: 'judge', model: 'gpt-6.1-sol', calls: 100, in_tokens: 300_000, out_tokens: 50_000, max_out_tokens: 2_048, scope: 'full' as const }, +]; + +function runGate(host: CapyHost, cwd: string, args: string[], extraEnv: Record = {}) { + const r = spawnSync(BIN, ['price-gate', ...args], { cwd, env: { ...host.env, ...extraEnv }, encoding: 'utf8' }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; +} + +describe('price-gate: the call-plan file format (formatEvalCalls / parseEvalCalls)', () => { + test('the reference emitter round-trips: binding lines, preconditions, call lines and the end line', () => { + const plan = { gate: GATE, head: 'a'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: 'b'.repeat(64) }], preconditions: ['ANTHROPIC_API_KEY set at run time'], rows: twoRows }; + const text = formatEvalCalls(plan); + expect(text).toContain(`GSTACK_EVAL_GATE v1: ${GATE}\n`); + expect(text).toContain('GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full\n'); + expect(text.trimEnd().split('\n').at(-1)).toBe('GSTACK_EVAL_CALLS v1 end: lines=2'); + const parsed = parseEvalCalls(text); + expect('plan' in parsed && parsed.plan).toEqual(plan); + }); + + test('a malformed, partial or unknown-version file refuses and names the line', () => { + const good = formatEvalCalls({ gate: GATE, head: 'a'.repeat(40), inputs: [], preconditions: [], rows: twoRows }); + const err = (text: string) => { const r = parseEvalCalls(text); return 'error' in r ? r.error : ''; }; + expect(err(good.replace(/\nGSTACK_EVAL_CALLS v1 end: lines=2\n$/, '\n'))).toMatch(/end line/); + expect(err(good.replace('end: lines=2', 'end: lines=3'))).toMatch(/lines=3.*2 call lines/); + expect(err(good.replace('calls=100 in_tokens=400000', 'calls=1e2 in_tokens=400000'))).toMatch(/line 4.*calls/); + expect(err(good.replace('scope=full\nGSTACK_EVAL_CALLS v1: purpose=judge', 'scope=full extra=1\nGSTACK_EVAL_CALLS v1: purpose=judge'))).toMatch(/line 4.*extra/); + expect(err(good.replace(' max_out_tokens=8192', ''))).toMatch(/line 4.*max_out_tokens/); + expect(err(good.replace('GSTACK_EVAL_CALLS v1: purpose=judge', 'GSTACK_EVAL_CALLS v2: purpose=judge'))).toMatch(/line 5.*version/); + expect(err(good + 'GSTACK_EVAL_PRECONDITION: after the end\n')).toMatch(/after the end line/); + expect(err(good.replace(/^GSTACK_EVAL_HEAD.*\n/m, ''))).toMatch(/GSTACK_EVAL_HEAD/); + }); +}); + +describe('price-gate: binding, pricing and the receipt (bin, simulated Capy host)', () => { + let host: CapyHost; + beforeEach(() => { host = capyHost(); }); + afterEach(() => host.cleanup()); + + function writePlan(repo: string, plan: Parameters[0]): string { + const file = path.join(host.base, 'plan.calls'); + fs.writeFileSync(file, formatEvalCalls(plan)); + return file; + } + + test('two call lines and one precondition are priced and listed, and the receipt binds price to gate, head, inputs and plan bytes', () => { + const { repo, head, rowsSha } = scratchRepo(host.env); + const file = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: ['ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time'], rows: twoRows }); + const out = path.join(host.base, 'receipts'); + const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '20', '--out', out]); + expect(r.err).toBe(''); + expect(r.code).toBe(0); + // opus-5-5: 0.4 MTok × $4 + 0.2 MTok × $20 = $5.60; sol: 0.3 × $2 + 0.05 × $10 = $1.10 + expect(r.out).toContain('GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full'); + expect(r.out).toContain('GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full'); + expect(r.out).toContain('GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time'); + const price = r.out.split('\n').find(l => l.startsWith('GATE_PRICE: '))!; + expect(price).toMatch(/^GATE_PRICE: total=6\.7 calls=200 purposes=answer,judge receipt=\S+$/); + const receiptPath = price.split('receipt=')[1]!; + expect(path.dirname(receiptPath)).toBe(out); + const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8')); + expect(receipt).toMatchObject({ + schema_version: 1, result: 'priced', code: null, gate_command: GATE, gate_sha256: createHash('sha256').update(GATE).digest('hex'), + head_sha: head, plan: { file, source: 'flag', declared_head: head, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') }, + inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], total_usd: 6.7, calls: 200, cap_usd: 20, unpriced_override: null, + }); + expect(typeof receipt.gstack_revision).toBe('string'); + }); + + test('the base policy price_file is read from the HEAD commit; a plan committed on top of its own head stays fresh until another file changes', () => { + const { repo, rowsSha } = scratchRepo(host.env); + const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim(); + fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/gate.calls' })); + git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy'); + const produced = git('rev-parse', 'HEAD'); + git('update-ref', 'refs/remotes/origin/main', produced); + fs.mkdirSync(path.join(repo, 'evals')); + fs.writeFileSync(path.join(repo, 'evals', 'gate.calls'), formatEvalCalls({ gate: GATE, head: produced, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows })); + git('add', 'evals/gate.calls'); git('commit', '-q', '-m', 'call plan'); + const out = path.join(host.base, 'receipts'); + let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]); + expect(r.err).toBe(''); + expect(r.code).toBe(0); + expect(r.out).toContain(`GATE_PLAN: file=evals/gate.calls source=price_file`); + expect(r.out).toContain('only the plan file changed since'); + const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8')); + expect(receipt).toMatchObject({ plan: { source: 'price_file', declared_head: produced }, head_sha: git('rev-parse', 'HEAD'), total_usd: 6.7 }); + fs.writeFileSync(path.join(repo, 'README.md'), 'later\n'); + git('add', 'README.md'); git('commit', '-q', '-m', 'later'); + r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/behind HEAD.*README\.md.*\(EVAL_PLAN_STALE\)/); + }); + + test('a declared price_file missing at HEAD refuses EVAL_PLAN_STALE (not overridable); an unresolvable base or an invalid pregate.json exits 3', () => { + const { repo } = scratchRepo(host.env); + const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim(); + const out = path.join(host.base, 'receipts'); + let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']); + expect(r.code).toBe(3); + expect(r.err).toContain('(POLICY_SOURCE_UNAVAILABLE)'); + fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true }); + fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/missing.calls' })); + git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy'); + git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD')); + r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']); + expect(r.code).toBe(1); + expect(r.err).toMatch(/evals\/missing\.calls.*\(EVAL_PLAN_STALE\)/); + expect(r.out).not.toContain('GATE_UNPRICED_OVERRIDE'); + fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: '../outside.calls' })); + git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'bad policy'); + git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD')); + r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']); + expect(r.code).toBe(3); + expect(r.err).toContain('(PREGATE_CONFIG_INVALID)'); + expect(fs.existsSync(out)).toBe(false); + }); + + test('an unknown or misspelled flag exits 2 instead of pricing without it', () => { + const { repo } = scratchRepo(host.env); + expect(runGate(host, repo, ['--gate', GATE, '--cap', '5']).code).toBe(2); + expect(runGate(host, repo, ['--cap-usd', '5']).code).toBe(2); + }); + + test('nothing in the plan file is executed: a gate command and precondition that would create markers leave none', () => { + const { repo, head, rowsSha } = scratchRepo(host.env); + const marker = path.join(host.base, 'gate-ran'); + const gate = `touch ${marker}`; + const file = writePlan(repo, { gate, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [`$(touch ${marker}-pre)`], rows: twoRows }); + const r = runGate(host, repo, ['--plan-file', file, '--gate', gate]); + expect(r.code).toBe(0); + expect(fs.existsSync(marker)).toBe(false); + expect(fs.existsSync(`${marker}-pre`)).toBe(false); + }); + + test('a stale head, an input-hash mismatch and a gate-command mismatch each refuse EVAL_PLAN_STALE with no receipt', () => { + const { repo, head, rowsSha } = scratchRepo(host.env); + const out = path.join(host.base, 'receipts'); + const staleHead = writePlan(repo, { gate: GATE, head: 'f'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows }); + let r = runGate(host, repo, ['--plan-file', staleHead, '--gate', GATE, '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/head .*ffffffffffff.*\(EVAL_PLAN_STALE\)/); + const badInput = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: '0'.repeat(64) }], preconditions: [], rows: twoRows }); + r = runGate(host, repo, ['--plan-file', badInput, '--gate', GATE, '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/rows\.jsonl.*sha256.*\(EVAL_PLAN_STALE\)/); + const good = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows }); + r = runGate(host, repo, ['--plan-file', good, '--gate', `${GATE} --questions 500`, '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/gate command.*\(EVAL_PLAN_STALE\)/); + expect(fs.existsSync(out)).toBe(false); + }); + + test('malformed, partial (no end line) and empty-of-binding files refuse EVAL_DRY_RUN_FAILED naming the line', () => { + const { repo, head } = scratchRepo(host.env); + const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }); + fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/GSTACK_EVAL_CALLS v1 end: lines=2\n$/, '')); + let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/end line.*\(EVAL_DRY_RUN_FAILED\)/); + fs.writeFileSync(file, formatEvalCalls({ gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }).replace('calls=100', 'calls=lots')); + r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]); + expect(r.code).toBe(1); + expect(r.err).toMatch(/line \d+.*calls.*\(EVAL_DRY_RUN_FAILED\)/); + }); + + test('no plan file, or a file with no call lines, is EVAL_GATE_UNPRICED exit 1; --allow-unpriced exits 0 with a recorded override', () => { + const { repo, head, rowsSha } = scratchRepo(host.env); + spawnSync('git', ['update-ref', 'refs/remotes/origin/main', head], { cwd: repo, env: host.env }); + const out = path.join(host.base, 'receipts'); + let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toContain('(EVAL_GATE_UNPRICED)'); + const empty = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: [] }); + r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out]); + expect(r.code).toBe(1); + expect(r.err).toContain('(EVAL_GATE_UNPRICED)'); + expect(r.out).toContain('GATE_PRICE: total=unknown calls=0 purposes=none'); + r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out, '--allow-unpriced', 'owner accepts a bounded smoke on a seat']); + expect(r.code).toBe(0); + expect(r.out).toContain('GATE_UNPRICED_OVERRIDE: owner accepts a bounded smoke on a seat (owner-authorized override; not authorization to spend)'); + const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8')); + expect(receipt).toMatchObject({ result: 'unpriced_override', code: 'EVAL_GATE_UNPRICED', unpriced_override: 'owner accepts a bounded smoke on a seat', total_usd: null }); + }); + + test('an unpriced model makes its row and the total unknown, never zero, and fails EVAL_PRICE_MISSING', () => { + const { repo, head } = scratchRepo(host.env); + const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [twoRows[0]!, { ...twoRows[1]!, model: 'acme/made-up-model' }] }); + const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]); + expect(r.code).toBe(1); + expect(r.out).toContain('GATE_CALLS: purpose=judge model=acme/made-up-model calls=100 usd=unknown reservation_usd=unknown scope=full'); + expect(r.out).toMatch(/GATE_PRICE: total=unknown calls=200 /); + expect(r.err).toContain('(EVAL_PRICE_MISSING)'); + }); + + test('a total above --cap-usd is EVAL_GATE_OVER_CAP; a single-call reservation above the cap or the smoke cap fails reservation-fits-cap', () => { + const { repo, head } = scratchRepo(host.env); + const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }); + let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '5']); + expect(r.code).toBe(1); + expect(r.err).toMatch(/6\.7.*cap \$5.*\(EVAL_GATE_OVER_CAP\)/); + const smoke = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [{ purpose: 'smoke-answer', model: 'claude-opus-5-5', calls: 2, in_tokens: 20_000, out_tokens: 4_000, max_out_tokens: 64_000, scope: 'smoke' }] }); + // one call: 10,000 in × $4 + 64,000 out × $20 per MTok = $1.32, under the $10 cap but over the $1 smoke cap + r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '10', '--smoke-cap-usd', '1']); + expect(r.code).toBe(1); + expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > smoke cap $1.00) purpose=smoke-answer'); + expect(r.err).toContain('(EVAL_GATE_OVER_CAP)'); + r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '1.2']); + expect(r.code).toBe(1); + expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > cap $1.20) purpose=smoke-answer'); + }); +}); + +describe('dry-run reservation-fits-cap', () => { + test('with --cap, one call per arm (largest row input + output budget), priced, must fit under the cap', () => { + const d = tmp(); + const rows = path.join(d, 'rows.jsonl'); + fs.writeFileSync(rows, [JSON.stringify({ prompt: 'x'.repeat(4_000) }), JSON.stringify({ prompt: 'x'.repeat(40_000) })].join('\n') + '\n'); + const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n'); + // largest row 10,000 tokens; one gpt-6-astra call = 10,000 × $10 + 8,192 × $50 per MTok = $0.5096 + const tight = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 0.5 }); + expect(tight.checks.reservation_fits_cap).toMatchObject({ ok: false, cap: 0.5, max_input_tokens: 10_000 }); + expect(tight.checks.reservation_fits_cap!.per_arm['gpt-6-astra']).toBeCloseTo(0.5096, 6); + expect(tight.pass).toBe(false); + expect(renderDryRun(tight)).toContain('CHECK: reservation-fits-cap exceeds ($0.5096 > cap $0.50)'); + const roomy = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 1 }); + expect(roomy.checks.reservation_fits_cap!.ok).toBe(true); + expect(roomy.pass).toBe(true); + expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry }).checks.reservation_fits_cap).toBeUndefined(); + }); +}); + +describe('pricing rows with fetched sources (2026-10-12)', () => { + test('the newest Sonnet, the newest Gemini Flash and gbrain embedding models are priced; Opus 4.7 is at its current list price', () => { + expect(hasPricing('anthropic/claude-sonnet-5-5')).toBe(true); + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-sonnet-5-5')).toBe(12); + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-opus-4-7')).toBe(30); + expect(hasPricing('google/gemini-3.8-flash')).toBe(true); + for (const m of ['voyage:voyage-4', 'openai:text-embedding-3-large', 'text-embedding-3-small', 'gemini-embedding-2']) expect(hasPricing(m)).toBe(true); + expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'voyage:voyage-4')).toBe(0.06); + expect(hasPricing('voyage-4-nano')).toBe(false); + }); +});