Merge remote-tracking branch 'origin/capy/gstack-field-wave-cd2' into capy/gstack-field-wave-cd

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

# Conflicts:
#	bin/gstack-doctor-components.sh
#	docs/troubleshooting.md
#	lib/result-codes.ts
This commit is contained in:
garrytan committed 2026-10-13 00:54:57 +00:00
commit 05e6f2e828
11 files changed
+907 -27

No files matched your search

+3
View File
@@ -168,6 +168,9 @@ gstack_result_anchor() {
DRAFT_DIRECTION_UNRESOLVED) printf '%s' 'draft-direction-unresolved' ;;
PREGATE_GUARD_FAILED) printf '%s' 'pregate-guard-failed' ;;
SEALED_OUTPUT_REFUSED) printf '%s' 'sealed-output-refused' ;;
EVAL_GATE_UNPRICED) printf '%s' 'eval-gate-unpriced' ;;
EVAL_GATE_OVER_CAP) printf '%s' 'eval-gate-over-cap' ;;
EVAL_PLAN_STALE) printf '%s' 'eval-plan-stale' ;;
*) return 1 ;;
esac
}
+52 -7
View File
@@ -10,20 +10,29 @@
* gstack-eval-plan pilot settle --out <dir> --attempt <id> --usd <n|unknown>
* gstack-eval-plan pilot release --out <dir> --attempt <id>
* gstack-eval-plan pilot report --out <dir> --total <items> [--workers W] [--json]
* gstack-eval-plan price-gate --gate "<exact gate command>" [--plan-file <file>] [--base <branch>] [--cap-usd N]
* [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out <dir>] [--json]
*
* The dry run spends nothing (stub model) and prints the checklist: price-table
* entry per arm, output budget sized for reasoning models, the retry prompt's
* exact text. The pilot reserves through lib/spend-ledger.ts (estimated admission,
* lock-shared refusal), settles actual cost, and `report` prints the spend table
* (spent, reserved, unknown charges, worst-case overrun) and the extrapolation
* with its error band before the approval ask. Exit 0 ok, 1 fail (incomplete
* prereg, failed check, exceeded cap), 2 usage, 3 refused. Logic: lib/eval-plan.ts.
* with its error band before the approval ask. `price-gate` prices a paid gate
* from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the
* base branch's .gstack/pregate.json read from the HEAD commit) after checking
* its binding (exact gate command, head, input hashes); it executes nothing and
* writes a receipt (docs/eval-calls.md). Exit 0 ok, 1 fail (incomplete
* prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused.
* Logic: lib/eval-plan.ts.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, releasePilot, renderArms, renderDryRun, renderPilotReport, resolveArms, settlePilot, startPilot } from '../lib/eval-plan';
import { spawnSync } from 'node:child_process';
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan';
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
import { resultLine } from '../lib/result-codes';
import { resolveBaseBranch } from '../lib/ship-policy';
import { SpendCapExceeded } from '../lib/spend-ledger';
const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md)
@@ -32,11 +41,16 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
prereg --init <file> [--title <t>] write the template
arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json]
arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item
dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--out dir] [--json]
stub model over the real rows; checklist: price table, output budget, retry prompt text
dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json]
stub model over the real rows; checklist: price table, output budget, retry prompt text,
and with --cap reservation-fits-cap (one call: largest row + output budget, priced)
pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3]
pilot settle --out dir --attempt id --usd <n|unknown> pilot release --out dir --attempt id
pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask
price-gate --gate "<exact gate command>" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out dir] [--json]
price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base
policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md
${exitTableHelp()}`;
@@ -85,13 +99,13 @@ function doDryRun(rest: string[]): number {
const arms = csv(flag(rest, '--arms'));
if (!input || !fs.existsSync(input)) return usage('dry-run needs --input <existing rows.jsonl>');
if (!arms.length) return usage('dry-run needs --arms <model,model>');
const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows') });
const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap') });
const out = flag(rest, '--out');
if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); }
if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r));
if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`);
if (!r.pass) {
const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : ''].filter(Boolean).join('; ');
const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : ''].filter(Boolean).join('; ');
process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n');
return EXIT.fail;
}
@@ -136,6 +150,36 @@ function doPilot(rest: string[]): number {
return usage(`unknown pilot command ${sub ?? ''}`);
}
const PRICE_GATE_FLAGS = new Set(['--gate', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out']);
function doPriceGate(rest: string[]): number {
for (let i = 0; i < rest.length; i++) {
if (rest[i] === '--json') continue;
if (!PRICE_GATE_FLAGS.has(rest[i]!) || rest[i + 1] === undefined) return usage(`price-gate: unknown flag or missing value ${rest[i]}`);
i++;
}
const gate = flag(rest, '--gate');
if (!gate) return usage('price-gate needs --gate "<exact gate command>"');
const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' });
if (top.status !== 0) return usage('price-gate runs inside the git repository whose gate it prices');
const repoRoot = top.stdout.trim();
const planFile = flag(rest, '--plan-file');
if (planFile && !fs.existsSync(planFile)) return usage(`--plan-file ${planFile} does not exist`);
const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) });
if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; }
const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') });
if (r.receipt) {
const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate'));
const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`);
fs.mkdirSync(out, { recursive: true });
fs.writeFileSync(file, JSON.stringify(r.receipt, null, 2) + '\n');
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r.receipt, receipt_path: file }) + '\n');
else process.stdout.write([...r.lines, renderGatePrice(r.receipt, file)].join('\n') + '\n');
}
if (r.error) { process.stderr.write(resultLine('gstack-eval-plan price-gate', r.error.code, r.error.message) + '\n'); return EXIT.fail; }
return EXIT.ok;
}
function main(argv: string[]): number {
const [cmd, ...rest] = argv;
if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; }
@@ -144,6 +188,7 @@ function main(argv: string[]): number {
if (cmd === 'arms') return doArms(rest);
if (cmd === 'dry-run') return doDryRun(rest);
if (cmd === 'pilot') return doPilot(rest);
if (cmd === 'price-gate') return doPriceGate(rest);
return usage(`unknown command ${cmd}`);
} catch (e: any) {
if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; }
+188
View File
@@ -0,0 +1,188 @@
# Eval call plans: pricing a paid gate without running it
A paid gate (a benchmark, an eval smoke, any command that calls a model
provider for money) is priced before anyone approves it. gstack does not run
the gate, or any other repository command, to learn its cost. The repository
commits a **call-plan file**: plain data that names the exact gate command, the
commit and inputs it was computed from, and one line per call purpose with its
model and token counts. `gstack-eval-plan price-gate` reads that file, checks
that it still describes what is being approved, prices it from
`lib/pricing.ts`, and writes a receipt. Plan: field wave CD2
(`docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md`, gate decision d5b as amended).
```bash
gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd 20 [--smoke-cap-usd 2] [--plan-file <file>]
```
## Where the file comes from
1. `--plan-file <file>`: an operator-given file, read from disk.
2. Otherwise `price_file` in `.gstack/pregate.json` as committed on
`origin/<base>` (`--base`, default origin/HEAD's branch): a repo-relative
path whose bytes are read from the **HEAD commit** (`git show HEAD:<path>`),
never from the working tree.
Nothing in the file is executed: not the gate command, not a precondition,
not an input path. The file is parsed as text.
## File format, wire version v1
```
# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed
GSTACK_EVAL_GATE v1: bun run eval:synopsis --questions 50 --arms opus,sol
GSTACK_EVAL_HEAD v1: 0123456789abcdef0123456789abcdef01234567
GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=evals/synopsis/questions.jsonl
GSTACK_EVAL_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time
GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full
GSTACK_EVAL_CALLS v1: purpose=judge model=gpt-6.1-sol calls=100 in_tokens=300000 out_tokens=50000 max_out_tokens=2048 scope=full
GSTACK_EVAL_CALLS v1 end: lines=2
```
| Line | Count | Meaning |
|---|---|---|
| `GSTACK_EVAL_GATE v1: <command>` | exactly 1 | the exact gate command this plan prices; compared to `--gate` by string equality |
| `GSTACK_EVAL_HEAD v1: <sha>` | exactly 1 | the full commit sha the plan was computed at |
| `GSTACK_EVAL_INPUT v1: sha256=<hex> path=<path>` | 0 or more | each input the counts came from (rows, datasets, prompts), repo-relative, no `..` |
| `GSTACK_EVAL_PRECONDITION: <text>` | 0 or more | what must hold at run time; every one is echoed as `GATE_PRECONDITION:` |
| `GSTACK_EVAL_CALLS v1: …` | 0 or more | one line per call purpose (unique `purpose` + `model`) |
| `GSTACK_EVAL_CALLS v1 end: lines=<n>` | exactly 1, last | `n` = the number of call lines; a missing or miscounted end line means a partial producer and refuses |
Blank lines and lines starting with `#` are ignored. Every other line must be
one of the above; an unknown kind, an unknown version (`v2`), a missing or
extra key, a duplicate key, a non-integer count, `calls=0`, or any content
after the end line refuses with `EVAL_DRY_RUN_FAILED`, naming the line.
### Call line keys
| Key | Meaning |
|---|---|
| `purpose` | name of the call purpose (`[A-Za-z0-9._:-]+`), for example `answer`, `judge`, `embed` |
| `model` | model id; a provider prefix (`anthropic/`, `openai:`, `voyage:`) is accepted |
| `calls` | number of calls for this purpose (at least 1) |
| `in_tokens` | **aggregate** input tokens across all of this purpose's calls |
| `out_tokens` | **aggregate** expected output tokens across all of this purpose's calls |
| `max_out_tokens` | the **per-call** output reservation (the `max_tokens` the call sends) |
| `scope` | `full` (the counted run) or `smoke` (a small pre-run check) |
The aggregate totals are what the purpose is expected to cost:
`usd = price(in_tokens, out_tokens)`. The per-call reservation is what one call
can cost at most, which is what a provider or a spend ledger reserves:
`reservation_usd = price(ceil(in_tokens / calls), max_out_tokens)`.
## Binding checks (`EVAL_PLAN_STALE`)
The plan refuses, with no receipt, when it no longer describes what is being
approved:
- the `GSTACK_EVAL_GATE` command differs from `--gate` by even one character;
- the `GSTACK_EVAL_HEAD` commit is not HEAD and any file other than the plan
file itself changed between them (a committed plan file sits one commit on
top of the head it was computed at, so its own commit is allowed);
- an input's sha256 of its bytes in the HEAD commit differs from its
`GSTACK_EVAL_INPUT` line, or the input is not in HEAD;
- the base policy declares a `price_file` that is not in the HEAD commit.
`--allow-unpriced` never overrides these: only a gate with no plan declared at
all, or a plan with zero call lines, is unpriced.
## Pricing outcomes
```
GATE_PLAN: file=evals/gate.calls source=price_file sha256=<hex> head=<sha12> inputs=1 verified
GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time
GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full
GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full
CHECK: reservation-fits-cap ok (largest one-call reservation $0.1798)
GATE_PRICE: total=6.7 calls=200 purposes=answer,judge receipt=.gstack/tmp/price-gate/price-gate-<gate12>-<head12>.json
```
| Outcome | Exit | Code |
|---|---|---|
| priced, under the caps | 0 | — |
| a model with no row in `lib/pricing.ts`: that row and the total print `unknown`, never zero | 1 | `EVAL_PRICE_MISSING` |
| total above `--cap-usd` | 1 | `EVAL_GATE_OVER_CAP` |
| one call's reservation above `--cap-usd`, or above `--smoke-cap-usd` for a `scope=smoke` row (`CHECK: reservation-fits-cap exceeds (…)`) | 1 | `EVAL_GATE_OVER_CAP` |
| no plan declared (no `--plan-file` and no `price_file` key) or a file with zero call lines | 1 | `EVAL_GATE_UNPRICED` |
| the same with `--allow-unpriced "<reason>"` | 0 | prints `GATE_UNPRICED_OVERRIDE: <reason> (owner-authorized override; not authorization to spend)` and records it |
| malformed or partial file | 1 | `EVAL_DRY_RUN_FAILED` |
| stale binding, or a declared `price_file` missing from HEAD (not overridable) | 1 | `EVAL_PLAN_STALE` |
| `origin/<base>` unresolvable while reading `price_file` | 3 | `POLICY_SOURCE_UNAVAILABLE` |
| `.gstack/pregate.json` on the base is invalid | 3 | `PREGATE_CONFIG_INVALID` |
| unknown flag or missing `--gate` | 2 | usage |
The price is an estimate: the table prices standard text tokens and excludes
the billing categories listed in `lib/pricing.ts` (cache writes, tool fees,
service tiers and long-context surcharges, media tokens, seat-metered runs).
## Receipt
Every outcome past the binding checks writes a JSON receipt to `--out`
(default `<repo>/.gstack/tmp/price-gate/`), named
`price-gate-<gate sha256 12>-<head 12>.json`, and prints its path on the
`GATE_PRICE:` line. `--json` prints the receipt (plus `receipt_path`) instead
of the lines.
| Field | Meaning |
|---|---|
| `schema_version` | `1` |
| `kind` | `gstack-eval-price-gate` |
| `created_at`, `gstack_revision` | when, and which gstack (VERSION and commit) priced it |
| `gate_command`, `gate_sha256` | the approved command and its sha256 |
| `head_sha` | HEAD when priced |
| `plan` | `file`, `source` (`flag`, `price_file`, `none`), `sha256` of the plan file bytes, `declared_head` |
| `inputs` | each `{ path, sha256 }`, verified at HEAD |
| `preconditions` | every precondition line |
| `rows` | each call line plus `usd` and `reservation_usd` (`null` when unpriced) |
| `total_usd`, `calls`, `purposes` | the totals; `total_usd` is `null` when unknown |
| `cap_usd`, `smoke_cap_usd` | the caps passed, or `null` |
| `result`, `code` | `priced`, `price_missing`, `over_cap`, `unpriced`, `unpriced_override`, and the result code |
| `unpriced_reason`, `unpriced_override` | why the gate is unpriced, and the owner's override reason |
| `excluded_billing` | the billing categories the estimate leaves out |
## Reference emitter
A consuming repo writes the file from the same code that builds its calls, so
the counts cannot drift from the run. `formatEvalCalls` in `lib/eval-plan.ts`
is the reference; copy it:
```ts
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
const KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const;
type Row = { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' };
export function formatEvalCalls(p: { gate: string; head: string; inputs: { path: string; sha256: string }[]; preconditions: string[]; rows: Row[] }): string {
return [
'# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed',
`GSTACK_EVAL_GATE v1: ${p.gate}`,
`GSTACK_EVAL_HEAD v1: ${p.head}`,
...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`),
...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`),
...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${KEYS.map(k => `${k}=${r[k]}`).join(' ')}`),
`GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`,
].join('\n') + '\n';
}
// Count tokens the way the run will, without calling a provider, then:
const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
const input = 'evals/synopsis/questions.jsonl';
const sha256 = createHash('sha256').update(readFileSync(input)).digest('hex');
// writeFileSync('evals/gate.calls', formatEvalCalls({ gate, head, inputs: [{ path: input, sha256 }], preconditions, rows }));
```
Hash the inputs as committed (the file at HEAD), produce the plan at a clean
HEAD, commit the plan file on its own commit, and set
`"price_file": "evals/gate.calls"` in `.gstack/pregate.json`.
## Related
- `gstack-eval-plan dry-run --cap <usd>` runs the same reservation-fits-cap
check for an eval's arms: one call (the largest row's input plus the output
budget), priced, must fit under the cap.
- Result codes: [EVAL_GATE_UNPRICED](troubleshooting.md#eval-gate-unpriced),
[EVAL_GATE_OVER_CAP](troubleshooting.md#eval-gate-over-cap),
[EVAL_PLAN_STALE](troubleshooting.md#eval-plan-stale),
[EVAL_PRICE_MISSING](troubleshooting.md#eval-price-missing),
[EVAL_DRY_RUN_FAILED](troubleshooting.md#eval-dry-run-failed).
+40
View File
@@ -2500,3 +2500,43 @@ it). A reviewed false positive gets one line in the base branch's
destination sets `"visibility": "private"` under `sealed` in the base branch's
`.gstack/pregate.json`. Capy's pull-request tool may not run local hooks, so
run `gstack-pregate sealed` yourself before any push or PR creation.
<a id="eval-gate-unpriced"></a>
### `EVAL_GATE_UNPRICED`
**Meaning.** `gstack-eval-plan price-gate` found no call-plan file for the paid
gate: no `--plan-file`, no `price_file` in the base branch's
`.gstack/pregate.json`, or the file has no `GSTACK_EVAL_CALLS` lines. (A
declared `price_file` missing from HEAD is `EVAL_PLAN_STALE`, not this.) A gate whose cost is unknown cannot
be approved as priced.
**Fix.** The consuming repo commits a call-plan file in the format of
[docs/eval-calls.md](eval-calls.md) and names it in `price_file`. Only when the
owner accepts an unpriced gate, rerun with `--allow-unpriced "<reason>"`: it
exits 0, prints `GATE_UNPRICED_OVERRIDE: <reason> (owner-authorized override;
not authorization to spend)` and records the reason in the receipt.
<a id="eval-gate-over-cap"></a>
### `EVAL_GATE_OVER_CAP`
**Meaning.** The gate's priced total is above `--cap-usd`, or one call's
reservation (its share of the input plus `max_out_tokens`) is above `--cap-usd`,
or above `--smoke-cap-usd` for a `scope=smoke` row. A provider refuses a
reservation the cap cannot hold, so the run would fail after spending.
**Fix.** Cut questions, arms or `max_out_tokens` in the call plan and
regenerate it, or raise the cap deliberately with the owner. Never split calls
to slip under it.
<a id="eval-plan-stale"></a>
### `EVAL_PLAN_STALE`
**Meaning.** The call-plan file prices something other than what is being
approved: its `GSTACK_EVAL_GATE` command is not the exact `--gate` string, its
`GSTACK_EVAL_HEAD` is behind HEAD with files other than the plan file changed
since, an input's sha256 at HEAD differs from its `GSTACK_EVAL_INPUT` line, or
the base policy's `price_file` is not in the HEAD commit. `--allow-unpriced`
does not override it.
**Fix.** Regenerate the plan file at HEAD for the exact gate command (the
repo's own producer), commit it, and rerun `gstack-eval-plan price-gate`.
+14 -1
View File
@@ -428,6 +428,8 @@ the budget the run will use. The checklist printed must be all `ok`:
spends output tokens thinking, and a smaller budget truncates the answer the scorer reads.
- `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry
with no text is a repeat of the same failure at the same price.
- `CHECK: reservation-fits-cap` (with `--cap <cap-usd>`) — one call, the largest row plus the
output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold.
- `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt.
`DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on
@@ -459,8 +461,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
## Step 4: The approval ask
Price every paid gate the plan names (a benchmark or eval command that spends) from the
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack):
```bash
~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd <cap-usd>
```
Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`,
`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced;
`--allow-unpriced "<reason>"` only on the owner's word.
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
question with the projection as the recommended option only when `verdict=within`:
- A) Run `<total-items>` items at the projected cost (recommended when within the cap)
+14 -1
View File
@@ -100,6 +100,8 @@ the budget the run will use. The checklist printed must be all `ok`:
spends output tokens thinking, and a smaller budget truncates the answer the scorer reads.
- `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry
with no text is a repeat of the same failure at the same price.
- `CHECK: reservation-fits-cap` (with `--cap <cap-usd>`) — one call, the largest row plus the
output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold.
- `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt.
`DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on
@@ -131,8 +133,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
## Step 4: The approval ask
Price every paid gate the plan names (a benchmark or eval command that spends) from the
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack):
```bash
~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd <cap-usd>
```
Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`,
`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced;
`--allow-unpriced "<reason>"` only on the owner's word.
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
question with the projection as the recommended option only when `verdict=within`:
- A) Run `<total-items>` items at the projected cost (recommended when within the cap)
+259 -5
View File
@@ -9,9 +9,12 @@
* (lib/pricing.ts), so the spend table always shows `unknown` charges and the
* worst-case overrun (one in-flight call per worker) beside `spent`.
*/
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { modelsMain } from './model-policy-cli';
import { isConfigUnavailable, loadConfigFromBase } from './pregate/config';
import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pricingRow } from './pricing';
import type { ResultCodeName } from './result-codes';
import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger';
@@ -129,10 +132,15 @@ export const REASONING_OUTPUT_FLOOR = 8_192;
export interface DryRunRow { line: number; ok: boolean; missing: string[]; input_tokens: number }
export interface DryRunResult {
schema_version: 1; input: string; rows: number; bad_rows: DryRunRow[]; est_input_tokens: number; est_output_tokens: number;
arms: string[]; checks: { price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null } };
arms: string[];
checks: {
price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null };
/** Present with `cap`: one call per arm (the largest row's input plus the output budget), priced, fits under the cap. */
reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record<string, number | null> };
};
per_item_usd: Record<string, number | null>; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)';
}
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number }
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number }
/** Four bytes per token, the estimator every gstack admission uses. */
export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4);
@@ -142,7 +150,7 @@ export function dryRun(o: DryRunOptions): DryRunResult {
const budget = o.outputBudget ?? REASONING_OUTPUT_FLOOR;
const text = fs.readFileSync(o.input, 'utf8');
const lines = text.split('\n');
const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0;
const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0; let maxInput = 0;
lines.forEach((line, i) => {
if (!line.trim()) return;
if (o.maxRows && rows >= o.maxRows) return;
@@ -153,6 +161,7 @@ export function dryRun(o: DryRunOptions): DryRunResult {
const missing = fields.filter(f => parsed![f] === undefined || parsed![f] === null || parsed![f] === '');
const t = estimateTokens(fields.map(f => typeof parsed![f] === 'string' ? parsed![f] as string : JSON.stringify(parsed![f] ?? '')).join('\n'));
inputTokens += t;
maxInput = Math.max(maxInput, t);
if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t });
});
const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null;
@@ -164,10 +173,15 @@ export function dryRun(o: DryRunOptions): DryRunResult {
price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice },
output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR },
retry_prompt: { ok: !!retryText, text: retryText },
};
} as DryRunResult['checks'];
if (o.cap !== undefined) {
const perArm: Record<string, number | null> = {};
for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null;
checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm };
}
return {
schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem,
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok, stub: 'stub-model (no network, $0)',
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true), stub: 'stub-model (no network, $0)',
};
}
@@ -178,12 +192,18 @@ export function renderDryRun(r: DryRunResult): string {
`CHECK: price-table ${r.checks.price_table.ok ? 'ok' : `missing ${r.checks.price_table.missing.join(',') || '(no arms)'}`}`,
`CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`,
`CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt <file> with the exact text the run will send)'}`,
...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []),
...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`),
`DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`,
];
return lines.join('\n') + '\n';
}
function renderReservationCheck(c: NonNullable<DryRunResult['checks']['reservation_fits_cap']>, budget: number): string {
const worst = Math.max(0, ...Object.values(c.per_arm).filter((v): v is number => v !== null));
return `CHECK: reservation-fits-cap ${c.ok ? 'ok' : 'exceeds'} ($${worst.toFixed(4)} ${c.ok ? '<=' : '>'} cap $${c.cap.toFixed(2)}) one call = ${c.max_input_tokens} in + ${budget} out tokens`;
}
// ---------------------------------------------------------------------------
// priced pilot over lib/spend-ledger.ts
// ---------------------------------------------------------------------------
@@ -275,3 +295,237 @@ export function renderPilotReport(r: PilotReport): string {
p.verdict === 'unknown' ? `APPROVAL: not ready — ${r.settled_known < 2 ? 'settle at least 2 pilot attempts with actual cost' : 'unknown charges make the band meaningless'}; settle, then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`,
].join('\n') + '\n';
}
// ---------------------------------------------------------------------------
// paid gates priced from a call-plan data file (docs/eval-calls.md)
// ---------------------------------------------------------------------------
/**
* The plan file is DATA: `price-gate` reads it, checks its binding (the exact
* gate command, the head it was produced at, each input's sha256 at HEAD) and
* prices its `GSTACK_EVAL_CALLS v1` rows. Nothing in it is ever executed.
*/
export const EVAL_CALL_KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const;
export interface EvalCallRow { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' }
export interface EvalPlanInput { path: string; sha256: string }
export interface EvalCallPlan { gate: string; head: string; inputs: EvalPlanInput[]; preconditions: string[]; rows: EvalCallRow[] }
/** The reference emitter a consuming repo copies: binding lines, preconditions, one line per call purpose, the end line. */
export function formatEvalCalls(p: EvalCallPlan): string {
return [
'# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed',
`GSTACK_EVAL_GATE v1: ${p.gate}`,
`GSTACK_EVAL_HEAD v1: ${p.head}`,
...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`),
...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`),
...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${EVAL_CALL_KEYS.map(k => `${k}=${r[k]}`).join(' ')}`),
`GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`,
].join('\n') + '\n';
}
const INT_KEYS = ['calls', 'in_tokens', 'out_tokens', 'max_out_tokens'] as const;
function parseCallRow(body: string, at: string): EvalCallRow | string {
const fields: Record<string, string> = {};
for (const tok of body.trim().split(/ +/)) {
const eq = tok.indexOf('=');
if (eq <= 0) return `${at}: ${JSON.stringify(tok)} is not key=value`;
const key = tok.slice(0, eq);
if (!(EVAL_CALL_KEYS as readonly string[]).includes(key)) return `${at}: unknown key ${key}`;
if (key in fields) return `${at}: duplicate key ${key}`;
fields[key] = tok.slice(eq + 1);
}
for (const key of EVAL_CALL_KEYS) if (!(key in fields)) return `${at}: missing key ${key}`;
for (const key of INT_KEYS) if (!/^\d+$/.test(fields[key]!)) return `${at}: ${key} must be a non-negative integer (got ${JSON.stringify(fields[key])})`;
if (Number(fields.calls) < 1) return `${at}: calls must be at least 1`;
if (fields.scope !== 'full' && fields.scope !== 'smoke') return `${at}: scope must be full or smoke (got ${JSON.stringify(fields.scope)})`;
if (!/^[A-Za-z0-9._:-]+$/.test(fields.purpose!)) return `${at}: purpose must match [A-Za-z0-9._:-]+`;
if (!fields.model) return `${at}: model is empty`;
return { purpose: fields.purpose!, model: fields.model, calls: Number(fields.calls), in_tokens: Number(fields.in_tokens), out_tokens: Number(fields.out_tokens), max_out_tokens: Number(fields.max_out_tokens), scope: fields.scope };
}
/** Strict v1 parser: a malformed line, an unknown kind or version, a missing binding or a missing or miscounted end line refuses, naming the line. */
export function parseEvalCalls(text: string): { plan: EvalCallPlan } | { error: string } {
const plan: EvalCallPlan = { gate: '', head: '', inputs: [], preconditions: [], rows: [] };
const seen = new Set<string>();
let gate: string | null = null; let head: string | null = null; let ended = false;
const lines = text.split('\n');
for (let i = 0; i < lines.length; i++) {
const raw = lines[i]!.replace(/\r$/, ''); const at = `line ${i + 1}`;
if (!raw.trim() || raw.startsWith('#')) continue;
if (ended) return { error: `${at}: content after the end line` };
const m = /^(GSTACK_EVAL_[A-Z_]+?)(?: (v\d+))?( end)?: (.*)$/.exec(raw);
if (!m) return { error: `${at}: not a GSTACK_EVAL_ line (${JSON.stringify(raw.slice(0, 80))})` };
const [, kind, version, end, body] = m as unknown as [string, string, string | undefined, string | undefined, string];
if (kind === 'GSTACK_EVAL_PRECONDITION' && !version && !end) { plan.preconditions.push(body.trim()); continue; }
if (!['GSTACK_EVAL_GATE', 'GSTACK_EVAL_HEAD', 'GSTACK_EVAL_INPUT', 'GSTACK_EVAL_CALLS'].includes(kind)) return { error: `${at}: unknown line kind ${kind}` };
if (version !== 'v1') return { error: `${at}: ${kind} version ${version ?? '(none)'} is not supported (this gstack reads v1)` };
if (end) {
const n = kind === 'GSTACK_EVAL_CALLS' ? /^lines=(\d+)$/.exec(body.trim()) : null;
if (!n) return { error: `${at}: malformed end line (expected GSTACK_EVAL_CALLS v1 end: lines=<n>)` };
if (Number(n[1]) !== plan.rows.length) return { error: `${at}: end line says lines=${n[1]} but the file has ${plan.rows.length} call lines` };
ended = true; continue;
}
if (kind === 'GSTACK_EVAL_GATE') {
if (gate !== null) return { error: `${at}: second GSTACK_EVAL_GATE line` };
if (!body.trim()) return { error: `${at}: empty gate command` };
gate = body; continue;
}
if (kind === 'GSTACK_EVAL_HEAD') {
if (head !== null) return { error: `${at}: second GSTACK_EVAL_HEAD line` };
if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/.test(body.trim())) return { error: `${at}: head must be a full commit sha` };
head = body.trim(); continue;
}
if (kind === 'GSTACK_EVAL_INPUT') {
const inp = /^sha256=([0-9a-f]{64}) path=(.+)$/.exec(body);
if (!inp) return { error: `${at}: expected GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=<repo-relative path>` };
if (path.isAbsolute(inp[2]!) || inp[2]!.split(/[\\/]/).includes('..')) return { error: `${at}: input path must be repo-relative without .. (${inp[2]})` };
plan.inputs.push({ path: inp[2]!, sha256: inp[1]! }); continue;
}
const row = parseCallRow(body, at);
if (typeof row === 'string') return { error: row };
if (seen.has(`${row.purpose}\0${row.model}`)) return { error: `${at}: duplicate purpose=${row.purpose} model=${row.model}` };
seen.add(`${row.purpose}\0${row.model}`);
plan.rows.push(row);
}
if (gate === null) return { error: 'missing GSTACK_EVAL_GATE v1 line (the exact gate command this plan prices)' };
if (head === null) return { error: 'missing GSTACK_EVAL_HEAD v1 line (the commit the plan was produced at)' };
if (!ended) return { error: 'missing end line (GSTACK_EVAL_CALLS v1 end: lines=<n>); a partial file is refused' };
return { plan: { ...plan, gate, head } };
}
export interface PricedCallRow extends EvalCallRow { usd: number | null; reservation_usd: number | null }
export interface GateReceipt {
schema_version: 1; kind: 'gstack-eval-price-gate'; created_at: string; gstack_revision: string;
gate_command: string; gate_sha256: string; head_sha: string;
plan: { file: string | null; source: 'flag' | 'price_file' | 'none'; sha256: string | null; declared_head: string | null };
inputs: EvalPlanInput[]; preconditions: string[]; rows: PricedCallRow[];
total_usd: number | null; calls: number; purposes: string[]; cap_usd: number | null; smoke_cap_usd: number | null;
result: 'priced' | 'price_missing' | 'over_cap' | 'unpriced' | 'unpriced_override'; code: ResultCodeName | null;
unpriced_reason: string | null; unpriced_override: string | null; excluded_billing: readonly string[];
}
export interface PlanSource { text: Buffer | null; file: string | null; source: GateReceipt['plan']['source']; repoPath: string | null; why: string | null }
export interface PriceGateOptions { repoRoot: string; gate: string; plan: PlanSource; capUsd?: number; smokeCapUsd?: number; allowUnpriced?: string; now?: Date }
export interface PriceGateResult { lines: string[]; receipt: GateReceipt | null; error?: { code: ResultCodeName; message: string } }
const sha256 = (b: Buffer | string) => createHash('sha256').update(b).digest('hex');
function gitOut(repoRoot: string, args: string[]): Buffer | null {
const r = spawnSync('git', args, { cwd: repoRoot, timeout: 30_000 });
return r.status === 0 ? r.stdout : null;
}
export interface PlanSourceRefusal { code: ResultCodeName; message: string; refused: boolean }
/**
* `--plan-file` wins; else the base policy's `price_file`, read from the HEAD
* commit's bytes. Only "no plan declared" is unpriced (overridable); a declared
* file missing at HEAD is stale, an unresolvable base or an invalid policy refuses.
*/
export function resolvePlanSource(repoRoot: string, o: { planFile?: string; base: string }): PlanSource | PlanSourceRefusal {
if (o.planFile) {
const file = path.resolve(o.planFile);
const rel = path.relative(repoRoot, file);
return { text: fs.readFileSync(file), file, source: 'flag', repoPath: rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel.split(path.sep).join('/') : null, why: null };
}
const load = loadConfigFromBase(repoRoot, o.base);
if (isConfigUnavailable(load)) return { code: 'POLICY_SOURCE_UNAVAILABLE', message: `no --plan-file and the base policy cannot be read (${load.detail})`, refused: true };
if (load.errors.length) return { code: 'PREGATE_CONFIG_INVALID', message: `${load.label}: ${load.errors.map(e => `${e.path}: ${e.message}`).join('; ')}`, refused: true };
const rel = load.config.price_file;
if (!rel) return { text: null, file: null, source: 'none', repoPath: null, why: `no --plan-file and ${load.label} declares no price_file` };
const text = gitOut(repoRoot, ['show', `HEAD:${rel}`]);
if (!text) return { code: 'EVAL_PLAN_STALE', message: `price_file ${rel} (declared in ${load.label}) is not in the HEAD commit`, refused: false };
return { text, file: rel, source: 'price_file', repoPath: rel, why: null };
}
/** Stale binding: the gate command, the head (only the plan file itself may change after it) and every input hash at HEAD. */
function checkBinding(o: PriceGateOptions, plan: EvalCallPlan, headSha: string): string | null {
if (plan.gate !== o.gate) return `gate command differs: --gate ${JSON.stringify(o.gate)} vs plan ${JSON.stringify(plan.gate)}`;
if (plan.head !== headSha) {
const diff = gitOut(o.repoRoot, ['diff', '--name-only', plan.head, headSha]);
if (diff === null) return `declared head ${plan.head.slice(0, 12)} is not a commit in this repository (HEAD ${headSha.slice(0, 12)})`;
const changed = diff.toString('utf8').split('\n').filter(Boolean);
if (changed.some(f => f !== o.plan.repoPath)) return `declared head ${plan.head.slice(0, 12)} is behind HEAD ${headSha.slice(0, 12)}: ${changed.length} file(s) changed since (${changed.slice(0, 3).join(', ')})`;
}
for (const input of plan.inputs) {
const bytes = gitOut(o.repoRoot, ['show', `HEAD:${input.path}`]);
if (!bytes) return `input ${input.path} is not in the HEAD commit`;
const got = sha256(bytes);
if (got !== input.sha256) return `input ${input.path} sha256 ${got.slice(0, 12)} at HEAD != declared ${input.sha256.slice(0, 12)}`;
}
return null;
}
export function gstackRevision(): string {
const root = path.resolve(import.meta.dir, '..');
const version = fs.existsSync(path.join(root, 'VERSION')) ? fs.readFileSync(path.join(root, 'VERSION'), 'utf8').trim() : 'unknown';
const sha = gitOut(root, ['rev-parse', '--short', 'HEAD'])?.toString('utf8').trim();
return sha ? `${version} (${sha})` : version;
}
const amount = (v: number | null) => (v === null ? 'unknown' : String(+v.toFixed(6)));
function priceRows(rows: EvalCallRow[]): PricedCallRow[] {
return rows.map(r => {
if (!hasPricing(r.model)) return { ...r, usd: null, reservation_usd: null };
return { ...r, usd: estimateCostUsd({ input: r.in_tokens, output: r.out_tokens }, r.model), reservation_usd: estimateCostUsd({ input: Math.ceil(r.in_tokens / r.calls), output: r.max_out_tokens }, r.model) };
});
}
/** Price a paid gate from its call-plan file. Malformed or stale files return an error and no receipt; every other outcome returns a receipt. */
export function priceGate(o: PriceGateOptions): PriceGateResult {
const headSha = gitOut(o.repoRoot, ['rev-parse', 'HEAD'])?.toString('utf8').trim() ?? '';
let plan: EvalCallPlan | null = null;
if (o.plan.text) {
const parsed = parseEvalCalls(o.plan.text.toString('utf8'));
if ('error' in parsed) return { lines: [], receipt: null, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.plan.file}: ${parsed.error}` } };
const stale = checkBinding(o, parsed.plan, headSha);
if (stale) return { lines: [], receipt: null, error: { code: 'EVAL_PLAN_STALE', message: `${o.plan.file}: ${stale}` } };
plan = parsed.plan;
}
const rows = priceRows(plan?.rows ?? []);
const total = rows.some(r => r.usd === null) ? null : +rows.reduce((s, r) => s + r.usd!, 0).toFixed(6);
const lines = [
...(plan ? [`GATE_PLAN: file=${o.plan.file} source=${o.plan.source} sha256=${sha256(o.plan.text!)} head=${plan.head.slice(0, 12)}${plan.head === headSha ? '' : ` (HEAD ${headSha.slice(0, 12)}; only the plan file changed since)`} inputs=${plan.inputs.length} verified`] : []),
...(plan?.preconditions ?? []).map(t => `GATE_PRECONDITION: ${t}`),
...rows.map(r => `GATE_CALLS: purpose=${r.purpose} model=${r.model} calls=${r.calls} usd=${amount(r.usd)} reservation_usd=${amount(r.reservation_usd)} scope=${r.scope}`),
];
let error: PriceGateResult['error'];
let result: GateReceipt['result'] = 'priced';
const unpricedWhy = !plan ? o.plan.why ?? 'no call-plan file' : rows.length === 0 ? `${o.plan.file} has no GSTACK_EVAL_CALLS lines` : null;
if (unpricedWhy) {
result = o.allowUnpriced ? 'unpriced_override' : 'unpriced';
if (o.allowUnpriced) lines.push(`GATE_UNPRICED_OVERRIDE: ${o.allowUnpriced} (owner-authorized override; not authorization to spend)`);
else error = { code: 'EVAL_GATE_UNPRICED', message: unpricedWhy };
} else if (total === null) {
result = 'price_missing';
error = { code: 'EVAL_PRICE_MISSING', message: `no price row for ${rows.filter(r => r.usd === null).map(r => r.model).join(', ')}; the total is unknown, never zero` };
} else {
const over: string[] = [];
if (o.capUsd !== undefined || o.smokeCapUsd !== undefined) {
for (const r of rows) {
const smoke = r.scope === 'smoke' && o.smokeCapUsd !== undefined && r.reservation_usd! > o.smokeCapUsd;
const full = o.capUsd !== undefined && r.reservation_usd! > o.capUsd;
if (!smoke && !full) continue;
const why = `$${r.reservation_usd!.toFixed(4)} > ${smoke ? `smoke cap $${o.smokeCapUsd!.toFixed(2)}` : `cap $${o.capUsd!.toFixed(2)}`}`;
lines.push(`CHECK: reservation-fits-cap exceeds (${why}) purpose=${r.purpose}`);
over.push(`reservation-fits-cap: purpose=${r.purpose} one call ${why}`);
}
if (!over.length) lines.push(`CHECK: reservation-fits-cap ok (largest one-call reservation $${Math.max(0, ...rows.map(r => r.reservation_usd!)).toFixed(4)})`);
}
if (o.capUsd !== undefined && total > o.capUsd) over.unshift(`total $${total.toFixed(4)} > cap $${o.capUsd.toFixed(2)}`);
if (over.length) { result = 'over_cap'; error = { code: 'EVAL_GATE_OVER_CAP', message: over.join('; ') }; }
}
const receipt: GateReceipt = {
schema_version: 1, kind: 'gstack-eval-price-gate', created_at: (o.now ?? new Date()).toISOString(), gstack_revision: gstackRevision(),
gate_command: o.gate, gate_sha256: sha256(o.gate), head_sha: headSha,
plan: { file: o.plan.file, source: o.plan.source, sha256: o.plan.text ? sha256(o.plan.text) : null, declared_head: plan?.head ?? null },
inputs: plan?.inputs ?? [], preconditions: plan?.preconditions ?? [], rows,
total_usd: unpricedWhy ? null : total, calls: rows.reduce((s, r) => s + r.calls, 0), purposes: rows.map(r => r.purpose),
cap_usd: o.capUsd ?? null, smoke_cap_usd: o.smokeCapUsd ?? null, result, code: error?.code ?? (unpricedWhy ? 'EVAL_GATE_UNPRICED' : null),
unpriced_reason: unpricedWhy, unpriced_override: unpricedWhy ? o.allowUnpriced ?? null : null, excluded_billing: EXCLUDED_BILLING_CATEGORIES,
};
return { lines, receipt, error };
}
export function renderGatePrice(r: GateReceipt, receiptPath: string): string {
return `GATE_PRICE: total=${amount(r.total_usd)} calls=${r.calls} purposes=${r.purposes.join(',') || 'none'} receipt=${receiptPath}`;
}
+40 -4
View File
@@ -20,7 +20,8 @@
* start a priced run on such a model (bin/gstack-eval-plan dry-run).
* - Anthropic: https://www.anthropic.com/pricing#api
* - OpenAI: https://openai.com/api/pricing/
* - Google AI: https://ai.google.dev/pricing
* - Google AI: https://ai.google.dev/gemini-api/docs/pricing
* - Voyage AI (embeddings): https://docs.voyageai.com/docs/pricing
*/
export interface ModelPricing {
@@ -45,7 +46,10 @@ export const PRICING: Record<string, ModelPricing> = {
'claude-fable-5-1': { input_per_mtok: 10.00, output_per_mtok: 50.00, cached_input_per_mtok: 0.25, as_of: '2026-09' },
// https://platform.claude.com/docs/en/models/opus-5-5/overview#pricing (list prices effective 2026-10-07)
'claude-opus-5-5': { input_per_mtok: 4.00, output_per_mtok: 20.00, cached_input_per_mtok: 0.20, as_of: '2026-10' },
'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' },
// https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12)
'claude-opus-4-7': { input_per_mtok: 5.00, output_per_mtok: 25.00, cached_input_per_mtok: 0.50, as_of: '2026-10' },
// https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12)
'claude-sonnet-5-5': { input_per_mtok: 2.00, output_per_mtok: 10.00, cached_input_per_mtok: 0.10, as_of: '2026-10' },
'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' },
'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' },
@@ -60,14 +64,46 @@ export const PRICING: Record<string, ModelPricing> = {
'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' },
// Google
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; standard tier through 2026-12-31,
// the page lists $1.50 / $7.50 / $0.15 cached starting 2027-01-01)
'gemini-3.8-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' },
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; same schedule as gemini-3.8-flash)
'gemini-3.6-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' },
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; prompts <= 200k tokens, the long-context rate is excluded)
'gemini-3.1-pro-preview': { input_per_mtok: 2.00, output_per_mtok: 12.00, cached_input_per_mtok: 0.20, as_of: '2026-10' },
'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' },
'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' },
};
/** A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`) resolves to its table row. */
/**
* Embedding models bill input tokens only (output_per_mtok 0), so they live
* beside PRICING rather than in it: eval arms never resolve to an embedding
* model, but a call plan (docs/eval-calls.md) prices gbrain's embedding calls.
*/
export const EMBEDDING_PRICING: Record<string, ModelPricing> = {
// https://docs.voyageai.com/docs/pricing (fetched 2026-10-12; list price after the free-token allowance)
'voyage-4': { input_per_mtok: 0.06, output_per_mtok: 0, as_of: '2026-10' },
'voyage-4-large': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' },
'voyage-4-lite': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' },
'voyage-code-4': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' },
'voyage-code-3': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' },
'voyage-3-large': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' },
'voyage-multimodal-3': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' }, // text tokens; pixels excluded
// https://developers.openai.com/api/docs/pricing (fetched 2026-10-12)
'text-embedding-3-large': { input_per_mtok: 0.13, output_per_mtok: 0, as_of: '2026-10' },
'text-embedding-3-small': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' },
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; text input, standard tier)
'gemini-embedding-2': { input_per_mtok: 0.20, output_per_mtok: 0, as_of: '2026-10' },
};
/**
* A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`,
* or gbrain's `voyage:voyage-4`) resolves to its table row.
*/
export function pricingRow(model: string | undefined): ModelPricing | undefined {
if (!model) return undefined;
return PRICING[model] ?? PRICING[model.replace(/^[a-z-]+\//, '')];
const bare = model.replace(/^[a-z-]+[/:]/, '');
return PRICING[model] ?? PRICING[bare] ?? EMBEDDING_PRICING[model] ?? EMBEDDING_PRICING[bare];
}
export function hasPricing(model: string | undefined): boolean {
+15
View File
@@ -476,6 +476,21 @@ export const RESULT_CODES = {
summary: 'a push or PR would publish sealed or eval output (a sealed path, a transcript, a quoted answer, a seed or a local home path) to a public or unknown destination, or the sealed policy is unreadable',
fix: 'remove it from every published commit, or add one reviewed `rule<TAB>path<TAB>sha<TAB>reason` line to the base branch\'s .gstack/sealed-allow.tsv; a private destination sets sealed.visibility "private" in the base branch\'s .gstack/pregate.json',
},
EVAL_GATE_UNPRICED: {
anchor: 'eval-gate-unpriced',
summary: 'the paid gate has no call-plan file (no `--plan-file`, no base-policy `price_file`) or the file has no GSTACK_EVAL_CALLS lines, so its cost is unknown',
fix: 'have the repo commit a call-plan file (docs/eval-calls.md) and set `price_file` in .gstack/pregate.json, or pass `--allow-unpriced "<reason>"` only on the owner\'s say-so',
},
EVAL_GATE_OVER_CAP: {
anchor: 'eval-gate-over-cap',
summary: 'the paid gate\'s priced total, or one call\'s reservation (input plus maximum output), is above the cap',
fix: 'cut questions, arms or `max_out_tokens` in the call plan, or raise `--cap-usd` / `--smoke-cap-usd` deliberately with the owner; never split calls to slip under it',
},
EVAL_PLAN_STALE: {
anchor: 'eval-plan-stale',
summary: 'the call-plan file\'s binding does not match (a different gate command, a head behind HEAD, an input whose sha256 at HEAD differs) or the declared price_file is not in HEAD',
fix: 'regenerate the call-plan file at HEAD for the exact gate command you are approving, commit it, and rerun `gstack-eval-plan price-gate`',
},
} as const satisfies Record<string, ResultCode>;
export type ResultCodeName = keyof typeof RESULT_CODES;
+7 -7
View File
@@ -23,10 +23,10 @@ test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
});
test('estimateCostUsd computes correctly for known Claude model', () => {
// claude-opus-4-7: $15/MTok input, $75/MTok output
// 1M input + 0.5M output = $15 + $37.50 = $52.50
// claude-opus-4-7: $5/MTok input, $25/MTok output (platform.claude.com pricing, fetched 2026-10-12)
// 1M input + 0.5M output = $5 + $12.50 = $17.50
const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7');
expect(cost).toBeCloseTo(52.50, 2);
expect(cost).toBeCloseTo(17.50, 2);
});
for (const model of [CLAUDE_FRONTIER_EVAL_MODEL, CODEX_FRONTIER_MODEL]) {
@@ -48,12 +48,12 @@ test('Astra cache reads use its standard cached-input rate', () => {
test('estimateCostUsd applies cached input discount alongside uncached input', () => {
// tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%.
// 0 uncached input, 1M cached → 10% of 15 = $1.50
// 0 uncached input, 1M cached → 10% of 5 = $0.50
const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7');
expect(cost1).toBeCloseTo(1.50, 2);
// 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25
expect(cost1).toBeCloseTo(0.50, 2);
// 500K uncached input + 500K cached → $2.50 + $0.25 = $2.75
const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7');
expect(cost2).toBeCloseTo(8.25, 2);
expect(cost2).toBeCloseTo(2.75, 2);
});
test('PRICING table covers the key model families', () => {
+275 -2
View File
@@ -3,16 +3,19 @@
* /eval-plan machinery in lib/eval-plan.ts: preregistration check, priced arms,
* the $0 dry run, and the priced pilot ledger with its spend table.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { EXCLUDED_BILLING_CATEGORIES, PRICING, estimateCostUsd, hasPricing, pricingRow } from '../lib/pricing';
import * as helperPricing from './helpers/pricing';
import {
PREREG_SECTIONS, checkPrereg, dryRun, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable,
PREREG_SECTIONS, checkPrereg, dryRun, formatEvalCalls, parseEvalCalls, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable,
resolveArms, settlePilot, startPilot,
} from '../lib/eval-plan';
import { capyHost, type CapyHost } from './helpers/capy-host';
const dirs: string[] = [];
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-plan-')); dirs.push(d); return d; };
@@ -156,3 +159,273 @@ describe('priced pilot', () => {
expect(r.projection.verdict).toBe('unknown');
});
});
// ---------------------------------------------------------------------------
// CD2 (field wave item 15): paid gates priced from a call-plan DATA file.
// `price-gate` never executes anything: it validates the file's binding (gate
// command, head, input hashes at HEAD) and prices the GSTACK_EVAL_CALLS v1 rows.
// ---------------------------------------------------------------------------
const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-eval-plan');
function scratchRepo(env: Record<string, string>): { repo: string; head: string; rowsSha: string } {
const repo = path.join(env.TMPDIR!, 'repo');
fs.mkdirSync(repo, { recursive: true });
const git = (...args: string[]) => {
const r = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8' });
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
return r.stdout.trim();
};
git('init', '-q', '-b', 'main');
fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n');
git('add', 'rows.jsonl');
git('commit', '-q', '-m', 'rows');
return { repo, head: git('rev-parse', 'HEAD'), rowsSha: createHash('sha256').update(fs.readFileSync(path.join(repo, 'rows.jsonl'))).digest('hex') };
}
const GATE = 'bun run eval:synopsis --questions 50 --arms opus,sol';
const twoRows = [
{ purpose: 'answer', model: 'anthropic/claude-opus-5-5', calls: 100, in_tokens: 400_000, out_tokens: 200_000, max_out_tokens: 8_192, scope: 'full' as const },
{ purpose: 'judge', model: 'gpt-6.1-sol', calls: 100, in_tokens: 300_000, out_tokens: 50_000, max_out_tokens: 2_048, scope: 'full' as const },
];
function runGate(host: CapyHost, cwd: string, args: string[], extraEnv: Record<string, string> = {}) {
const r = spawnSync(BIN, ['price-gate', ...args], { cwd, env: { ...host.env, ...extraEnv }, encoding: 'utf8' });
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
}
describe('price-gate: the call-plan file format (formatEvalCalls / parseEvalCalls)', () => {
test('the reference emitter round-trips: binding lines, preconditions, call lines and the end line', () => {
const plan = { gate: GATE, head: 'a'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: 'b'.repeat(64) }], preconditions: ['ANTHROPIC_API_KEY set at run time'], rows: twoRows };
const text = formatEvalCalls(plan);
expect(text).toContain(`GSTACK_EVAL_GATE v1: ${GATE}\n`);
expect(text).toContain('GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full\n');
expect(text.trimEnd().split('\n').at(-1)).toBe('GSTACK_EVAL_CALLS v1 end: lines=2');
const parsed = parseEvalCalls(text);
expect('plan' in parsed && parsed.plan).toEqual(plan);
});
test('a malformed, partial or unknown-version file refuses and names the line', () => {
const good = formatEvalCalls({ gate: GATE, head: 'a'.repeat(40), inputs: [], preconditions: [], rows: twoRows });
const err = (text: string) => { const r = parseEvalCalls(text); return 'error' in r ? r.error : ''; };
expect(err(good.replace(/\nGSTACK_EVAL_CALLS v1 end: lines=2\n$/, '\n'))).toMatch(/end line/);
expect(err(good.replace('end: lines=2', 'end: lines=3'))).toMatch(/lines=3.*2 call lines/);
expect(err(good.replace('calls=100 in_tokens=400000', 'calls=1e2 in_tokens=400000'))).toMatch(/line 4.*calls/);
expect(err(good.replace('scope=full\nGSTACK_EVAL_CALLS v1: purpose=judge', 'scope=full extra=1\nGSTACK_EVAL_CALLS v1: purpose=judge'))).toMatch(/line 4.*extra/);
expect(err(good.replace(' max_out_tokens=8192', ''))).toMatch(/line 4.*max_out_tokens/);
expect(err(good.replace('GSTACK_EVAL_CALLS v1: purpose=judge', 'GSTACK_EVAL_CALLS v2: purpose=judge'))).toMatch(/line 5.*version/);
expect(err(good + 'GSTACK_EVAL_PRECONDITION: after the end\n')).toMatch(/after the end line/);
expect(err(good.replace(/^GSTACK_EVAL_HEAD.*\n/m, ''))).toMatch(/GSTACK_EVAL_HEAD/);
});
});
describe('price-gate: binding, pricing and the receipt (bin, simulated Capy host)', () => {
let host: CapyHost;
beforeEach(() => { host = capyHost(); });
afterEach(() => host.cleanup());
function writePlan(repo: string, plan: Parameters<typeof formatEvalCalls>[0]): string {
const file = path.join(host.base, 'plan.calls');
fs.writeFileSync(file, formatEvalCalls(plan));
return file;
}
test('two call lines and one precondition are priced and listed, and the receipt binds price to gate, head, inputs and plan bytes', () => {
const { repo, head, rowsSha } = scratchRepo(host.env);
const file = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: ['ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time'], rows: twoRows });
const out = path.join(host.base, 'receipts');
const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '20', '--out', out]);
expect(r.err).toBe('');
expect(r.code).toBe(0);
// opus-5-5: 0.4 MTok × $4 + 0.2 MTok × $20 = $5.60; sol: 0.3 × $2 + 0.05 × $10 = $1.10
expect(r.out).toContain('GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full');
expect(r.out).toContain('GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full');
expect(r.out).toContain('GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time');
const price = r.out.split('\n').find(l => l.startsWith('GATE_PRICE: '))!;
expect(price).toMatch(/^GATE_PRICE: total=6\.7 calls=200 purposes=answer,judge receipt=\S+$/);
const receiptPath = price.split('receipt=')[1]!;
expect(path.dirname(receiptPath)).toBe(out);
const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8'));
expect(receipt).toMatchObject({
schema_version: 1, result: 'priced', code: null, gate_command: GATE, gate_sha256: createHash('sha256').update(GATE).digest('hex'),
head_sha: head, plan: { file, source: 'flag', declared_head: head, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') },
inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], total_usd: 6.7, calls: 200, cap_usd: 20, unpriced_override: null,
});
expect(typeof receipt.gstack_revision).toBe('string');
});
test('the base policy price_file is read from the HEAD commit; a plan committed on top of its own head stays fresh until another file changes', () => {
const { repo, rowsSha } = scratchRepo(host.env);
const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim();
fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/gate.calls' }));
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
const produced = git('rev-parse', 'HEAD');
git('update-ref', 'refs/remotes/origin/main', produced);
fs.mkdirSync(path.join(repo, 'evals'));
fs.writeFileSync(path.join(repo, 'evals', 'gate.calls'), formatEvalCalls({ gate: GATE, head: produced, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows }));
git('add', 'evals/gate.calls'); git('commit', '-q', '-m', 'call plan');
const out = path.join(host.base, 'receipts');
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
expect(r.err).toBe('');
expect(r.code).toBe(0);
expect(r.out).toContain(`GATE_PLAN: file=evals/gate.calls source=price_file`);
expect(r.out).toContain('only the plan file changed since');
const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
expect(receipt).toMatchObject({ plan: { source: 'price_file', declared_head: produced }, head_sha: git('rev-parse', 'HEAD'), total_usd: 6.7 });
fs.writeFileSync(path.join(repo, 'README.md'), 'later\n');
git('add', 'README.md'); git('commit', '-q', '-m', 'later');
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/behind HEAD.*README\.md.*\(EVAL_PLAN_STALE\)/);
});
test('a declared price_file missing at HEAD refuses EVAL_PLAN_STALE (not overridable); an unresolvable base or an invalid pregate.json exits 3', () => {
const { repo } = scratchRepo(host.env);
const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim();
const out = path.join(host.base, 'receipts');
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
expect(r.code).toBe(3);
expect(r.err).toContain('(POLICY_SOURCE_UNAVAILABLE)');
fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/missing.calls' }));
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
expect(r.code).toBe(1);
expect(r.err).toMatch(/evals\/missing\.calls.*\(EVAL_PLAN_STALE\)/);
expect(r.out).not.toContain('GATE_UNPRICED_OVERRIDE');
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: '../outside.calls' }));
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'bad policy');
git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
expect(r.code).toBe(3);
expect(r.err).toContain('(PREGATE_CONFIG_INVALID)');
expect(fs.existsSync(out)).toBe(false);
});
test('an unknown or misspelled flag exits 2 instead of pricing without it', () => {
const { repo } = scratchRepo(host.env);
expect(runGate(host, repo, ['--gate', GATE, '--cap', '5']).code).toBe(2);
expect(runGate(host, repo, ['--cap-usd', '5']).code).toBe(2);
});
test('nothing in the plan file is executed: a gate command and precondition that would create markers leave none', () => {
const { repo, head, rowsSha } = scratchRepo(host.env);
const marker = path.join(host.base, 'gate-ran');
const gate = `touch ${marker}`;
const file = writePlan(repo, { gate, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [`$(touch ${marker}-pre)`], rows: twoRows });
const r = runGate(host, repo, ['--plan-file', file, '--gate', gate]);
expect(r.code).toBe(0);
expect(fs.existsSync(marker)).toBe(false);
expect(fs.existsSync(`${marker}-pre`)).toBe(false);
});
test('a stale head, an input-hash mismatch and a gate-command mismatch each refuse EVAL_PLAN_STALE with no receipt', () => {
const { repo, head, rowsSha } = scratchRepo(host.env);
const out = path.join(host.base, 'receipts');
const staleHead = writePlan(repo, { gate: GATE, head: 'f'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
let r = runGate(host, repo, ['--plan-file', staleHead, '--gate', GATE, '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/head .*ffffffffffff.*\(EVAL_PLAN_STALE\)/);
const badInput = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: '0'.repeat(64) }], preconditions: [], rows: twoRows });
r = runGate(host, repo, ['--plan-file', badInput, '--gate', GATE, '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/rows\.jsonl.*sha256.*\(EVAL_PLAN_STALE\)/);
const good = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
r = runGate(host, repo, ['--plan-file', good, '--gate', `${GATE} --questions 500`, '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/gate command.*\(EVAL_PLAN_STALE\)/);
expect(fs.existsSync(out)).toBe(false);
});
test('malformed, partial (no end line) and empty-of-binding files refuse EVAL_DRY_RUN_FAILED naming the line', () => {
const { repo, head } = scratchRepo(host.env);
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/GSTACK_EVAL_CALLS v1 end: lines=2\n$/, ''));
let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/end line.*\(EVAL_DRY_RUN_FAILED\)/);
fs.writeFileSync(file, formatEvalCalls({ gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }).replace('calls=100', 'calls=lots'));
r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
expect(r.code).toBe(1);
expect(r.err).toMatch(/line \d+.*calls.*\(EVAL_DRY_RUN_FAILED\)/);
});
test('no plan file, or a file with no call lines, is EVAL_GATE_UNPRICED exit 1; --allow-unpriced exits 0 with a recorded override', () => {
const { repo, head, rowsSha } = scratchRepo(host.env);
spawnSync('git', ['update-ref', 'refs/remotes/origin/main', head], { cwd: repo, env: host.env });
const out = path.join(host.base, 'receipts');
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
const empty = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: [] });
r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out]);
expect(r.code).toBe(1);
expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
expect(r.out).toContain('GATE_PRICE: total=unknown calls=0 purposes=none');
r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out, '--allow-unpriced', 'owner accepts a bounded smoke on a seat']);
expect(r.code).toBe(0);
expect(r.out).toContain('GATE_UNPRICED_OVERRIDE: owner accepts a bounded smoke on a seat (owner-authorized override; not authorization to spend)');
const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
expect(receipt).toMatchObject({ result: 'unpriced_override', code: 'EVAL_GATE_UNPRICED', unpriced_override: 'owner accepts a bounded smoke on a seat', total_usd: null });
});
test('an unpriced model makes its row and the total unknown, never zero, and fails EVAL_PRICE_MISSING', () => {
const { repo, head } = scratchRepo(host.env);
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [twoRows[0]!, { ...twoRows[1]!, model: 'acme/made-up-model' }] });
const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
expect(r.code).toBe(1);
expect(r.out).toContain('GATE_CALLS: purpose=judge model=acme/made-up-model calls=100 usd=unknown reservation_usd=unknown scope=full');
expect(r.out).toMatch(/GATE_PRICE: total=unknown calls=200 /);
expect(r.err).toContain('(EVAL_PRICE_MISSING)');
});
test('a total above --cap-usd is EVAL_GATE_OVER_CAP; a single-call reservation above the cap or the smoke cap fails reservation-fits-cap', () => {
const { repo, head } = scratchRepo(host.env);
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '5']);
expect(r.code).toBe(1);
expect(r.err).toMatch(/6\.7.*cap \$5.*\(EVAL_GATE_OVER_CAP\)/);
const smoke = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [{ purpose: 'smoke-answer', model: 'claude-opus-5-5', calls: 2, in_tokens: 20_000, out_tokens: 4_000, max_out_tokens: 64_000, scope: 'smoke' }] });
// one call: 10,000 in × $4 + 64,000 out × $20 per MTok = $1.32, under the $10 cap but over the $1 smoke cap
r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '10', '--smoke-cap-usd', '1']);
expect(r.code).toBe(1);
expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > smoke cap $1.00) purpose=smoke-answer');
expect(r.err).toContain('(EVAL_GATE_OVER_CAP)');
r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '1.2']);
expect(r.code).toBe(1);
expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > cap $1.20) purpose=smoke-answer');
});
});
describe('dry-run reservation-fits-cap', () => {
test('with --cap, one call per arm (largest row input + output budget), priced, must fit under the cap', () => {
const d = tmp();
const rows = path.join(d, 'rows.jsonl');
fs.writeFileSync(rows, [JSON.stringify({ prompt: 'x'.repeat(4_000) }), JSON.stringify({ prompt: 'x'.repeat(40_000) })].join('\n') + '\n');
const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
// largest row 10,000 tokens; one gpt-6-astra call = 10,000 × $10 + 8,192 × $50 per MTok = $0.5096
const tight = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 0.5 });
expect(tight.checks.reservation_fits_cap).toMatchObject({ ok: false, cap: 0.5, max_input_tokens: 10_000 });
expect(tight.checks.reservation_fits_cap!.per_arm['gpt-6-astra']).toBeCloseTo(0.5096, 6);
expect(tight.pass).toBe(false);
expect(renderDryRun(tight)).toContain('CHECK: reservation-fits-cap exceeds ($0.5096 > cap $0.50)');
const roomy = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 1 });
expect(roomy.checks.reservation_fits_cap!.ok).toBe(true);
expect(roomy.pass).toBe(true);
expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry }).checks.reservation_fits_cap).toBeUndefined();
});
});
describe('pricing rows with fetched sources (2026-10-12)', () => {
test('the newest Sonnet, the newest Gemini Flash and gbrain embedding models are priced; Opus 4.7 is at its current list price', () => {
expect(hasPricing('anthropic/claude-sonnet-5-5')).toBe(true);
expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-sonnet-5-5')).toBe(12);
expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-opus-4-7')).toBe(30);
expect(hasPricing('google/gemini-3.8-flash')).toBe(true);
for (const m of ['voyage:voyage-4', 'openai:text-embedding-3-large', 'text-embedding-3-small', 'gemini-embedding-2']) expect(hasPricing(m)).toBe(true);
expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'voyage:voyage-4')).toBe(0.06);
expect(hasPricing('voyage-4-nano')).toBe(false);
});
});