mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
Merge remote-tracking branch 'origin/capy/gstack-field-wave-cd2' into capy/gstack-field-wave-cd
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # bin/gstack-doctor-components.sh # docs/troubleshooting.md # lib/result-codes.ts
This commit is contained in:
commit
05e6f2e828
11 files changed
+907
-27
No files matched your search
@@ -168,6 +168,9 @@ gstack_result_anchor() {
|
||||
DRAFT_DIRECTION_UNRESOLVED) printf '%s' 'draft-direction-unresolved' ;;
|
||||
PREGATE_GUARD_FAILED) printf '%s' 'pregate-guard-failed' ;;
|
||||
SEALED_OUTPUT_REFUSED) printf '%s' 'sealed-output-refused' ;;
|
||||
EVAL_GATE_UNPRICED) printf '%s' 'eval-gate-unpriced' ;;
|
||||
EVAL_GATE_OVER_CAP) printf '%s' 'eval-gate-over-cap' ;;
|
||||
EVAL_PLAN_STALE) printf '%s' 'eval-plan-stale' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
+52
-7
@@ -10,20 +10,29 @@
|
||||
* gstack-eval-plan pilot settle --out <dir> --attempt <id> --usd <n|unknown>
|
||||
* gstack-eval-plan pilot release --out <dir> --attempt <id>
|
||||
* gstack-eval-plan pilot report --out <dir> --total <items> [--workers W] [--json]
|
||||
* gstack-eval-plan price-gate --gate "<exact gate command>" [--plan-file <file>] [--base <branch>] [--cap-usd N]
|
||||
* [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out <dir>] [--json]
|
||||
*
|
||||
* The dry run spends nothing (stub model) and prints the checklist: price-table
|
||||
* entry per arm, output budget sized for reasoning models, the retry prompt's
|
||||
* exact text. The pilot reserves through lib/spend-ledger.ts (estimated admission,
|
||||
* lock-shared refusal), settles actual cost, and `report` prints the spend table
|
||||
* (spent, reserved, unknown charges, worst-case overrun) and the extrapolation
|
||||
* with its error band before the approval ask. Exit 0 ok, 1 fail (incomplete
|
||||
* prereg, failed check, exceeded cap), 2 usage, 3 refused. Logic: lib/eval-plan.ts.
|
||||
* with its error band before the approval ask. `price-gate` prices a paid gate
|
||||
* from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the
|
||||
* base branch's .gstack/pregate.json read from the HEAD commit) after checking
|
||||
* its binding (exact gate command, head, input hashes); it executes nothing and
|
||||
* writes a receipt (docs/eval-calls.md). Exit 0 ok, 1 fail (incomplete
|
||||
* prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused.
|
||||
* Logic: lib/eval-plan.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, releasePilot, renderArms, renderDryRun, renderPilotReport, resolveArms, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
import { resolveBaseBranch } from '../lib/ship-policy';
|
||||
import { SpendCapExceeded } from '../lib/spend-ledger';
|
||||
|
||||
const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md)
|
||||
@@ -32,11 +41,16 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
|
||||
prereg --init <file> [--title <t>] write the template
|
||||
arms [--arms m,m] [--policy-doc f] [--input-tokens N] [--output-tokens N] [--json]
|
||||
arms from \`gstack-models resolve --role eval-arms\`, else the repo policy doc, each priced per item
|
||||
dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--out dir] [--json]
|
||||
stub model over the real rows; checklist: price table, output budget, retry prompt text
|
||||
dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json]
|
||||
stub model over the real rows; checklist: price table, output budget, retry prompt text,
|
||||
and with --cap reservation-fits-cap (one call: largest row + output budget, priced)
|
||||
pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3]
|
||||
pilot settle --out dir --attempt id --usd <n|unknown> pilot release --out dir --attempt id
|
||||
pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask
|
||||
price-gate --gate "<exact gate command>" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out dir] [--json]
|
||||
price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base
|
||||
policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
|
||||
receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md
|
||||
|
||||
${exitTableHelp()}`;
|
||||
|
||||
@@ -85,13 +99,13 @@ function doDryRun(rest: string[]): number {
|
||||
const arms = csv(flag(rest, '--arms'));
|
||||
if (!input || !fs.existsSync(input)) return usage('dry-run needs --input <existing rows.jsonl>');
|
||||
if (!arms.length) return usage('dry-run needs --arms <model,model>');
|
||||
const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows') });
|
||||
const r = dryRun({ input, arms, fields: csv(flag(rest, '--fields')), outputBudget: num(rest, '--output-budget'), retryPrompt: flag(rest, '--retry-prompt'), maxRows: num(rest, '--max-rows'), cap: num(rest, '--cap') });
|
||||
const out = flag(rest, '--out');
|
||||
if (out) { fs.mkdirSync(out, { recursive: true }); fs.writeFileSync(path.join(out, 'dry-run.json'), JSON.stringify(r, null, 2) + '\n'); }
|
||||
if (rest.includes('--json')) process.stdout.write(JSON.stringify(r) + '\n'); else process.stdout.write(renderDryRun(r));
|
||||
if (out) process.stdout.write(`DRY_RUN_RECORD: ${path.join(out, 'dry-run.json')}\n`);
|
||||
if (!r.pass) {
|
||||
const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : ''].filter(Boolean).join('; ');
|
||||
const why = [r.rows === 0 ? 'no rows' : '', r.bad_rows.length ? `${r.bad_rows.length} bad rows` : '', !r.checks.price_table.ok ? `price missing: ${r.checks.price_table.missing.join(',') || 'no arms'}` : '', !r.checks.output_budget.ok ? 'output budget below the reasoning floor' : '', !r.checks.retry_prompt.ok ? 'retry prompt missing' : '', r.checks.reservation_fits_cap && !r.checks.reservation_fits_cap.ok ? `one call exceeds --cap $${r.checks.reservation_fits_cap.cap}` : ''].filter(Boolean).join('; ');
|
||||
process.stderr.write(resultLine('gstack-eval-plan dry-run', r.checks.price_table.ok ? 'EVAL_DRY_RUN_FAILED' : 'EVAL_PRICE_MISSING', why) + '\n');
|
||||
return EXIT.fail;
|
||||
}
|
||||
@@ -136,6 +150,36 @@ function doPilot(rest: string[]): number {
|
||||
return usage(`unknown pilot command ${sub ?? ''}`);
|
||||
}
|
||||
|
||||
const PRICE_GATE_FLAGS = new Set(['--gate', '--plan-file', '--base', '--cap-usd', '--smoke-cap-usd', '--allow-unpriced', '--out']);
|
||||
|
||||
function doPriceGate(rest: string[]): number {
|
||||
for (let i = 0; i < rest.length; i++) {
|
||||
if (rest[i] === '--json') continue;
|
||||
if (!PRICE_GATE_FLAGS.has(rest[i]!) || rest[i + 1] === undefined) return usage(`price-gate: unknown flag or missing value ${rest[i]}`);
|
||||
i++;
|
||||
}
|
||||
const gate = flag(rest, '--gate');
|
||||
if (!gate) return usage('price-gate needs --gate "<exact gate command>"');
|
||||
const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' });
|
||||
if (top.status !== 0) return usage('price-gate runs inside the git repository whose gate it prices');
|
||||
const repoRoot = top.stdout.trim();
|
||||
const planFile = flag(rest, '--plan-file');
|
||||
if (planFile && !fs.existsSync(planFile)) return usage(`--plan-file ${planFile} does not exist`);
|
||||
const plan = resolvePlanSource(repoRoot, { planFile, base: resolveBaseBranch(repoRoot, flag(rest, '--base')) });
|
||||
if ('code' in plan) { process.stderr.write(resultLine('gstack-eval-plan price-gate', plan.code, plan.message) + '\n'); return plan.refused ? EXIT.refused : EXIT.fail; }
|
||||
const r = priceGate({ repoRoot, gate, plan, capUsd: num(rest, '--cap-usd'), smokeCapUsd: num(rest, '--smoke-cap-usd'), allowUnpriced: flag(rest, '--allow-unpriced') });
|
||||
if (r.receipt) {
|
||||
const out = path.resolve(flag(rest, '--out') ?? path.join(repoRoot, '.gstack', 'tmp', 'price-gate'));
|
||||
const file = path.join(out, `price-gate-${r.receipt.gate_sha256.slice(0, 12)}-${r.receipt.head_sha.slice(0, 12)}.json`);
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify(r.receipt, null, 2) + '\n');
|
||||
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ ...r.receipt, receipt_path: file }) + '\n');
|
||||
else process.stdout.write([...r.lines, renderGatePrice(r.receipt, file)].join('\n') + '\n');
|
||||
}
|
||||
if (r.error) { process.stderr.write(resultLine('gstack-eval-plan price-gate', r.error.code, r.error.message) + '\n'); return EXIT.fail; }
|
||||
return EXIT.ok;
|
||||
}
|
||||
|
||||
function main(argv: string[]): number {
|
||||
const [cmd, ...rest] = argv;
|
||||
if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; }
|
||||
@@ -144,6 +188,7 @@ function main(argv: string[]): number {
|
||||
if (cmd === 'arms') return doArms(rest);
|
||||
if (cmd === 'dry-run') return doDryRun(rest);
|
||||
if (cmd === 'pilot') return doPilot(rest);
|
||||
if (cmd === 'price-gate') return doPriceGate(rest);
|
||||
return usage(`unknown command ${cmd}`);
|
||||
} catch (e: any) {
|
||||
if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; }
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# Eval call plans: pricing a paid gate without running it
|
||||
|
||||
A paid gate (a benchmark, an eval smoke, any command that calls a model
|
||||
provider for money) is priced before anyone approves it. gstack does not run
|
||||
the gate, or any other repository command, to learn its cost. The repository
|
||||
commits a **call-plan file**: plain data that names the exact gate command, the
|
||||
commit and inputs it was computed from, and one line per call purpose with its
|
||||
model and token counts. `gstack-eval-plan price-gate` reads that file, checks
|
||||
that it still describes what is being approved, prices it from
|
||||
`lib/pricing.ts`, and writes a receipt. Plan: field wave CD2
|
||||
(`docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md`, gate decision d5b as amended).
|
||||
|
||||
```bash
|
||||
gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd 20 [--smoke-cap-usd 2] [--plan-file <file>]
|
||||
```
|
||||
|
||||
## Where the file comes from
|
||||
|
||||
1. `--plan-file <file>`: an operator-given file, read from disk.
|
||||
2. Otherwise `price_file` in `.gstack/pregate.json` as committed on
|
||||
`origin/<base>` (`--base`, default origin/HEAD's branch): a repo-relative
|
||||
path whose bytes are read from the **HEAD commit** (`git show HEAD:<path>`),
|
||||
never from the working tree.
|
||||
|
||||
Nothing in the file is executed: not the gate command, not a precondition,
|
||||
not an input path. The file is parsed as text.
|
||||
|
||||
## File format, wire version v1
|
||||
|
||||
```
|
||||
# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed
|
||||
GSTACK_EVAL_GATE v1: bun run eval:synopsis --questions 50 --arms opus,sol
|
||||
GSTACK_EVAL_HEAD v1: 0123456789abcdef0123456789abcdef01234567
|
||||
GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=evals/synopsis/questions.jsonl
|
||||
GSTACK_EVAL_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time
|
||||
GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full
|
||||
GSTACK_EVAL_CALLS v1: purpose=judge model=gpt-6.1-sol calls=100 in_tokens=300000 out_tokens=50000 max_out_tokens=2048 scope=full
|
||||
GSTACK_EVAL_CALLS v1 end: lines=2
|
||||
```
|
||||
|
||||
| Line | Count | Meaning |
|
||||
|---|---|---|
|
||||
| `GSTACK_EVAL_GATE v1: <command>` | exactly 1 | the exact gate command this plan prices; compared to `--gate` by string equality |
|
||||
| `GSTACK_EVAL_HEAD v1: <sha>` | exactly 1 | the full commit sha the plan was computed at |
|
||||
| `GSTACK_EVAL_INPUT v1: sha256=<hex> path=<path>` | 0 or more | each input the counts came from (rows, datasets, prompts), repo-relative, no `..` |
|
||||
| `GSTACK_EVAL_PRECONDITION: <text>` | 0 or more | what must hold at run time; every one is echoed as `GATE_PRECONDITION:` |
|
||||
| `GSTACK_EVAL_CALLS v1: …` | 0 or more | one line per call purpose (unique `purpose` + `model`) |
|
||||
| `GSTACK_EVAL_CALLS v1 end: lines=<n>` | exactly 1, last | `n` = the number of call lines; a missing or miscounted end line means a partial producer and refuses |
|
||||
|
||||
Blank lines and lines starting with `#` are ignored. Every other line must be
|
||||
one of the above; an unknown kind, an unknown version (`v2`), a missing or
|
||||
extra key, a duplicate key, a non-integer count, `calls=0`, or any content
|
||||
after the end line refuses with `EVAL_DRY_RUN_FAILED`, naming the line.
|
||||
|
||||
### Call line keys
|
||||
|
||||
| Key | Meaning |
|
||||
|---|---|
|
||||
| `purpose` | name of the call purpose (`[A-Za-z0-9._:-]+`), for example `answer`, `judge`, `embed` |
|
||||
| `model` | model id; a provider prefix (`anthropic/`, `openai:`, `voyage:`) is accepted |
|
||||
| `calls` | number of calls for this purpose (at least 1) |
|
||||
| `in_tokens` | **aggregate** input tokens across all of this purpose's calls |
|
||||
| `out_tokens` | **aggregate** expected output tokens across all of this purpose's calls |
|
||||
| `max_out_tokens` | the **per-call** output reservation (the `max_tokens` the call sends) |
|
||||
| `scope` | `full` (the counted run) or `smoke` (a small pre-run check) |
|
||||
|
||||
The aggregate totals are what the purpose is expected to cost:
|
||||
`usd = price(in_tokens, out_tokens)`. The per-call reservation is what one call
|
||||
can cost at most, which is what a provider or a spend ledger reserves:
|
||||
`reservation_usd = price(ceil(in_tokens / calls), max_out_tokens)`.
|
||||
|
||||
## Binding checks (`EVAL_PLAN_STALE`)
|
||||
|
||||
The plan refuses, with no receipt, when it no longer describes what is being
|
||||
approved:
|
||||
|
||||
- the `GSTACK_EVAL_GATE` command differs from `--gate` by even one character;
|
||||
- the `GSTACK_EVAL_HEAD` commit is not HEAD and any file other than the plan
|
||||
file itself changed between them (a committed plan file sits one commit on
|
||||
top of the head it was computed at, so its own commit is allowed);
|
||||
- an input's sha256 of its bytes in the HEAD commit differs from its
|
||||
`GSTACK_EVAL_INPUT` line, or the input is not in HEAD;
|
||||
- the base policy declares a `price_file` that is not in the HEAD commit.
|
||||
|
||||
`--allow-unpriced` never overrides these: only a gate with no plan declared at
|
||||
all, or a plan with zero call lines, is unpriced.
|
||||
|
||||
## Pricing outcomes
|
||||
|
||||
```
|
||||
GATE_PLAN: file=evals/gate.calls source=price_file sha256=<hex> head=<sha12> inputs=1 verified
|
||||
GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time
|
||||
GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full
|
||||
GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full
|
||||
CHECK: reservation-fits-cap ok (largest one-call reservation $0.1798)
|
||||
GATE_PRICE: total=6.7 calls=200 purposes=answer,judge receipt=.gstack/tmp/price-gate/price-gate-<gate12>-<head12>.json
|
||||
```
|
||||
|
||||
| Outcome | Exit | Code |
|
||||
|---|---|---|
|
||||
| priced, under the caps | 0 | — |
|
||||
| a model with no row in `lib/pricing.ts`: that row and the total print `unknown`, never zero | 1 | `EVAL_PRICE_MISSING` |
|
||||
| total above `--cap-usd` | 1 | `EVAL_GATE_OVER_CAP` |
|
||||
| one call's reservation above `--cap-usd`, or above `--smoke-cap-usd` for a `scope=smoke` row (`CHECK: reservation-fits-cap exceeds (…)`) | 1 | `EVAL_GATE_OVER_CAP` |
|
||||
| no plan declared (no `--plan-file` and no `price_file` key) or a file with zero call lines | 1 | `EVAL_GATE_UNPRICED` |
|
||||
| the same with `--allow-unpriced "<reason>"` | 0 | prints `GATE_UNPRICED_OVERRIDE: <reason> (owner-authorized override; not authorization to spend)` and records it |
|
||||
| malformed or partial file | 1 | `EVAL_DRY_RUN_FAILED` |
|
||||
| stale binding, or a declared `price_file` missing from HEAD (not overridable) | 1 | `EVAL_PLAN_STALE` |
|
||||
| `origin/<base>` unresolvable while reading `price_file` | 3 | `POLICY_SOURCE_UNAVAILABLE` |
|
||||
| `.gstack/pregate.json` on the base is invalid | 3 | `PREGATE_CONFIG_INVALID` |
|
||||
| unknown flag or missing `--gate` | 2 | usage |
|
||||
|
||||
The price is an estimate: the table prices standard text tokens and excludes
|
||||
the billing categories listed in `lib/pricing.ts` (cache writes, tool fees,
|
||||
service tiers and long-context surcharges, media tokens, seat-metered runs).
|
||||
|
||||
## Receipt
|
||||
|
||||
Every outcome past the binding checks writes a JSON receipt to `--out`
|
||||
(default `<repo>/.gstack/tmp/price-gate/`), named
|
||||
`price-gate-<gate sha256 12>-<head 12>.json`, and prints its path on the
|
||||
`GATE_PRICE:` line. `--json` prints the receipt (plus `receipt_path`) instead
|
||||
of the lines.
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `schema_version` | `1` |
|
||||
| `kind` | `gstack-eval-price-gate` |
|
||||
| `created_at`, `gstack_revision` | when, and which gstack (VERSION and commit) priced it |
|
||||
| `gate_command`, `gate_sha256` | the approved command and its sha256 |
|
||||
| `head_sha` | HEAD when priced |
|
||||
| `plan` | `file`, `source` (`flag`, `price_file`, `none`), `sha256` of the plan file bytes, `declared_head` |
|
||||
| `inputs` | each `{ path, sha256 }`, verified at HEAD |
|
||||
| `preconditions` | every precondition line |
|
||||
| `rows` | each call line plus `usd` and `reservation_usd` (`null` when unpriced) |
|
||||
| `total_usd`, `calls`, `purposes` | the totals; `total_usd` is `null` when unknown |
|
||||
| `cap_usd`, `smoke_cap_usd` | the caps passed, or `null` |
|
||||
| `result`, `code` | `priced`, `price_missing`, `over_cap`, `unpriced`, `unpriced_override`, and the result code |
|
||||
| `unpriced_reason`, `unpriced_override` | why the gate is unpriced, and the owner's override reason |
|
||||
| `excluded_billing` | the billing categories the estimate leaves out |
|
||||
|
||||
## Reference emitter
|
||||
|
||||
A consuming repo writes the file from the same code that builds its calls, so
|
||||
the counts cannot drift from the run. `formatEvalCalls` in `lib/eval-plan.ts`
|
||||
is the reference; copy it:
|
||||
|
||||
```ts
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
const KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const;
|
||||
type Row = { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' };
|
||||
|
||||
export function formatEvalCalls(p: { gate: string; head: string; inputs: { path: string; sha256: string }[]; preconditions: string[]; rows: Row[] }): string {
|
||||
return [
|
||||
'# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed',
|
||||
`GSTACK_EVAL_GATE v1: ${p.gate}`,
|
||||
`GSTACK_EVAL_HEAD v1: ${p.head}`,
|
||||
...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`),
|
||||
...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`),
|
||||
...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${KEYS.map(k => `${k}=${r[k]}`).join(' ')}`),
|
||||
`GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`,
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
|
||||
// Count tokens the way the run will, without calling a provider, then:
|
||||
const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
|
||||
const input = 'evals/synopsis/questions.jsonl';
|
||||
const sha256 = createHash('sha256').update(readFileSync(input)).digest('hex');
|
||||
// writeFileSync('evals/gate.calls', formatEvalCalls({ gate, head, inputs: [{ path: input, sha256 }], preconditions, rows }));
|
||||
```
|
||||
|
||||
Hash the inputs as committed (the file at HEAD), produce the plan at a clean
|
||||
HEAD, commit the plan file on its own commit, and set
|
||||
`"price_file": "evals/gate.calls"` in `.gstack/pregate.json`.
|
||||
|
||||
## Related
|
||||
|
||||
- `gstack-eval-plan dry-run --cap <usd>` runs the same reservation-fits-cap
|
||||
check for an eval's arms: one call (the largest row's input plus the output
|
||||
budget), priced, must fit under the cap.
|
||||
- Result codes: [EVAL_GATE_UNPRICED](troubleshooting.md#eval-gate-unpriced),
|
||||
[EVAL_GATE_OVER_CAP](troubleshooting.md#eval-gate-over-cap),
|
||||
[EVAL_PLAN_STALE](troubleshooting.md#eval-plan-stale),
|
||||
[EVAL_PRICE_MISSING](troubleshooting.md#eval-price-missing),
|
||||
[EVAL_DRY_RUN_FAILED](troubleshooting.md#eval-dry-run-failed).
|
||||
@@ -2500,3 +2500,43 @@ it). A reviewed false positive gets one line in the base branch's
|
||||
destination sets `"visibility": "private"` under `sealed` in the base branch's
|
||||
`.gstack/pregate.json`. Capy's pull-request tool may not run local hooks, so
|
||||
run `gstack-pregate sealed` yourself before any push or PR creation.
|
||||
|
||||
<a id="eval-gate-unpriced"></a>
|
||||
### `EVAL_GATE_UNPRICED`
|
||||
|
||||
**Meaning.** `gstack-eval-plan price-gate` found no call-plan file for the paid
|
||||
gate: no `--plan-file`, no `price_file` in the base branch's
|
||||
`.gstack/pregate.json`, or the file has no `GSTACK_EVAL_CALLS` lines. (A
|
||||
declared `price_file` missing from HEAD is `EVAL_PLAN_STALE`, not this.) A gate whose cost is unknown cannot
|
||||
be approved as priced.
|
||||
|
||||
**Fix.** The consuming repo commits a call-plan file in the format of
|
||||
[docs/eval-calls.md](eval-calls.md) and names it in `price_file`. Only when the
|
||||
owner accepts an unpriced gate, rerun with `--allow-unpriced "<reason>"`: it
|
||||
exits 0, prints `GATE_UNPRICED_OVERRIDE: <reason> (owner-authorized override;
|
||||
not authorization to spend)` and records the reason in the receipt.
|
||||
|
||||
<a id="eval-gate-over-cap"></a>
|
||||
### `EVAL_GATE_OVER_CAP`
|
||||
|
||||
**Meaning.** The gate's priced total is above `--cap-usd`, or one call's
|
||||
reservation (its share of the input plus `max_out_tokens`) is above `--cap-usd`,
|
||||
or above `--smoke-cap-usd` for a `scope=smoke` row. A provider refuses a
|
||||
reservation the cap cannot hold, so the run would fail after spending.
|
||||
|
||||
**Fix.** Cut questions, arms or `max_out_tokens` in the call plan and
|
||||
regenerate it, or raise the cap deliberately with the owner. Never split calls
|
||||
to slip under it.
|
||||
|
||||
<a id="eval-plan-stale"></a>
|
||||
### `EVAL_PLAN_STALE`
|
||||
|
||||
**Meaning.** The call-plan file prices something other than what is being
|
||||
approved: its `GSTACK_EVAL_GATE` command is not the exact `--gate` string, its
|
||||
`GSTACK_EVAL_HEAD` is behind HEAD with files other than the plan file changed
|
||||
since, an input's sha256 at HEAD differs from its `GSTACK_EVAL_INPUT` line, or
|
||||
the base policy's `price_file` is not in the HEAD commit. `--allow-unpriced`
|
||||
does not override it.
|
||||
|
||||
**Fix.** Regenerate the plan file at HEAD for the exact gate command (the
|
||||
repo's own producer), commit it, and rerun `gstack-eval-plan price-gate`.
|
||||
+14
-1
@@ -428,6 +428,8 @@ the budget the run will use. The checklist printed must be all `ok`:
|
||||
spends output tokens thinking, and a smaller budget truncates the answer the scorer reads.
|
||||
- `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry
|
||||
with no text is a repeat of the same failure at the same price.
|
||||
- `CHECK: reservation-fits-cap` (with `--cap <cap-usd>`) — one call, the largest row plus the
|
||||
output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold.
|
||||
- `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt.
|
||||
|
||||
`DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on
|
||||
@@ -459,8 +461,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
|
||||
|
||||
## Step 4: The approval ask
|
||||
|
||||
Price every paid gate the plan names (a benchmark or eval command that spends) from the
|
||||
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack):
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd <cap-usd>
|
||||
```
|
||||
|
||||
Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`,
|
||||
`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced;
|
||||
`--allow-unpriced "<reason>"` only on the owner's word.
|
||||
|
||||
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
|
||||
table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
|
||||
table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
|
||||
question with the projection as the recommended option only when `verdict=within`:
|
||||
|
||||
- A) Run `<total-items>` items at the projected cost (recommended when within the cap)
|
||||
|
||||
+14
-1
@@ -100,6 +100,8 @@ the budget the run will use. The checklist printed must be all `ok`:
|
||||
spends output tokens thinking, and a smaller budget truncates the answer the scorer reads.
|
||||
- `CHECK: retry-prompt` — the exact text the run sends on a retry, printed verbatim; a retry
|
||||
with no text is a repeat of the same failure at the same price.
|
||||
- `CHECK: reservation-fits-cap` (with `--cap <cap-usd>`) — one call, the largest row plus the
|
||||
output budget, priced, fits under the cap; a provider refuses a reservation the cap cannot hold.
|
||||
- `DRY_RUN_ROW:` lines name rows missing a field; fix the rows, not the prompt.
|
||||
|
||||
`DRY_RUN_RESULT: pass` writes `dry-run.json` to `--out`; the pilot refuses to start on
|
||||
@@ -131,8 +133,19 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
|
||||
|
||||
## Step 4: The approval ask
|
||||
|
||||
Price every paid gate the plan names (a benchmark or eval command that spends) from the
|
||||
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack):
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan price-gate --gate "<exact gate command>" --cap-usd <cap-usd>
|
||||
```
|
||||
|
||||
Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STALE`,
|
||||
`EVAL_PRICE_MISSING` or `EVAL_GATE_OVER_CAP` means that gate is not approvable as priced;
|
||||
`--allow-unpriced "<reason>"` only on the owner's word.
|
||||
|
||||
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
|
||||
table, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
|
||||
table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
|
||||
question with the projection as the recommended option only when `verdict=within`:
|
||||
|
||||
- A) Run `<total-items>` items at the projected cost (recommended when within the cap)
|
||||
|
||||
+259
-5
@@ -9,9 +9,12 @@
|
||||
* (lib/pricing.ts), so the spend table always shows `unknown` charges and the
|
||||
* worst-case overrun (one in-flight call per worker) beside `spent`.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { modelsMain } from './model-policy-cli';
|
||||
import { isConfigUnavailable, loadConfigFromBase } from './pregate/config';
|
||||
import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pricingRow } from './pricing';
|
||||
import type { ResultCodeName } from './result-codes';
|
||||
import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger';
|
||||
@@ -129,10 +132,15 @@ export const REASONING_OUTPUT_FLOOR = 8_192;
|
||||
export interface DryRunRow { line: number; ok: boolean; missing: string[]; input_tokens: number }
|
||||
export interface DryRunResult {
|
||||
schema_version: 1; input: string; rows: number; bad_rows: DryRunRow[]; est_input_tokens: number; est_output_tokens: number;
|
||||
arms: string[]; checks: { price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null } };
|
||||
arms: string[];
|
||||
checks: {
|
||||
price_table: { ok: boolean; missing: string[] }; output_budget: { ok: boolean; budget: number; floor: number }; retry_prompt: { ok: boolean; text: string | null };
|
||||
/** Present with `cap`: one call per arm (the largest row's input plus the output budget), priced, fits under the cap. */
|
||||
reservation_fits_cap?: { ok: boolean; cap: number; max_input_tokens: number; per_arm: Record<string, number | null> };
|
||||
};
|
||||
per_item_usd: Record<string, number | null>; excluded_billing: readonly string[]; pass: boolean; stub: 'stub-model (no network, $0)';
|
||||
}
|
||||
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number }
|
||||
export interface DryRunOptions { input: string; arms: string[]; fields?: string[]; outputBudget?: number; retryPrompt?: string; maxRows?: number; cap?: number }
|
||||
|
||||
/** Four bytes per token, the estimator every gstack admission uses. */
|
||||
export const estimateTokens = (text: string) => Math.ceil(Buffer.byteLength(text, 'utf8') / 4);
|
||||
@@ -142,7 +150,7 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
const budget = o.outputBudget ?? REASONING_OUTPUT_FLOOR;
|
||||
const text = fs.readFileSync(o.input, 'utf8');
|
||||
const lines = text.split('\n');
|
||||
const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0;
|
||||
const bad: DryRunRow[] = []; let rows = 0; let inputTokens = 0; let maxInput = 0;
|
||||
lines.forEach((line, i) => {
|
||||
if (!line.trim()) return;
|
||||
if (o.maxRows && rows >= o.maxRows) return;
|
||||
@@ -153,6 +161,7 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
const missing = fields.filter(f => parsed![f] === undefined || parsed![f] === null || parsed![f] === '');
|
||||
const t = estimateTokens(fields.map(f => typeof parsed![f] === 'string' ? parsed![f] as string : JSON.stringify(parsed![f] ?? '')).join('\n'));
|
||||
inputTokens += t;
|
||||
maxInput = Math.max(maxInput, t);
|
||||
if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t });
|
||||
});
|
||||
const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null;
|
||||
@@ -164,10 +173,15 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice },
|
||||
output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR },
|
||||
retry_prompt: { ok: !!retryText, text: retryText },
|
||||
};
|
||||
} as DryRunResult['checks'];
|
||||
if (o.cap !== undefined) {
|
||||
const perArm: Record<string, number | null> = {};
|
||||
for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null;
|
||||
checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm };
|
||||
}
|
||||
return {
|
||||
schema_version: 1, input: o.input, rows, bad_rows: bad, est_input_tokens: inputTokens, est_output_tokens: rows * budget, arms: o.arms, checks, per_item_usd: perItem,
|
||||
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok, stub: 'stub-model (no network, $0)',
|
||||
excluded_billing: EXCLUDED_BILLING_CATEGORIES, pass: rows > 0 && bad.length === 0 && checks.price_table.ok && checks.output_budget.ok && checks.retry_prompt.ok && (checks.reservation_fits_cap?.ok ?? true), stub: 'stub-model (no network, $0)',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,12 +192,18 @@ export function renderDryRun(r: DryRunResult): string {
|
||||
`CHECK: price-table ${r.checks.price_table.ok ? 'ok' : `missing ${r.checks.price_table.missing.join(',') || '(no arms)'}`}`,
|
||||
`CHECK: output-budget ${r.checks.output_budget.ok ? 'ok' : 'too small'} (${r.checks.output_budget.budget} tokens; reasoning floor ${r.checks.output_budget.floor})`,
|
||||
`CHECK: retry-prompt ${r.checks.retry_prompt.ok ? `ok — ${JSON.stringify(r.checks.retry_prompt.text)}` : 'missing (pass --retry-prompt <file> with the exact text the run will send)'}`,
|
||||
...(r.checks.reservation_fits_cap ? [renderReservationCheck(r.checks.reservation_fits_cap, r.checks.output_budget.budget)] : []),
|
||||
...Object.entries(r.per_item_usd).map(([m, usd]) => `ESTIMATE: ${m} per_item=${usd === null ? 'unknown (no price)' : `$${usd.toFixed(4)}`} (estimate: excludes ${r.excluded_billing.length} billing categories, see lib/pricing.ts)`),
|
||||
`DRY_RUN_RESULT: ${r.pass ? 'pass' : 'fail'}`,
|
||||
];
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function renderReservationCheck(c: NonNullable<DryRunResult['checks']['reservation_fits_cap']>, budget: number): string {
|
||||
const worst = Math.max(0, ...Object.values(c.per_arm).filter((v): v is number => v !== null));
|
||||
return `CHECK: reservation-fits-cap ${c.ok ? 'ok' : 'exceeds'} ($${worst.toFixed(4)} ${c.ok ? '<=' : '>'} cap $${c.cap.toFixed(2)}) one call = ${c.max_input_tokens} in + ${budget} out tokens`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// priced pilot over lib/spend-ledger.ts
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -275,3 +295,237 @@ export function renderPilotReport(r: PilotReport): string {
|
||||
p.verdict === 'unknown' ? `APPROVAL: not ready — ${r.settled_known < 2 ? 'settle at least 2 pilot attempts with actual cost' : 'unknown charges make the band meaningless'}; settle, then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`,
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// paid gates priced from a call-plan data file (docs/eval-calls.md)
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* The plan file is DATA: `price-gate` reads it, checks its binding (the exact
|
||||
* gate command, the head it was produced at, each input's sha256 at HEAD) and
|
||||
* prices its `GSTACK_EVAL_CALLS v1` rows. Nothing in it is ever executed.
|
||||
*/
|
||||
export const EVAL_CALL_KEYS = ['purpose', 'model', 'calls', 'in_tokens', 'out_tokens', 'max_out_tokens', 'scope'] as const;
|
||||
export interface EvalCallRow { purpose: string; model: string; calls: number; in_tokens: number; out_tokens: number; max_out_tokens: number; scope: 'full' | 'smoke' }
|
||||
export interface EvalPlanInput { path: string; sha256: string }
|
||||
export interface EvalCallPlan { gate: string; head: string; inputs: EvalPlanInput[]; preconditions: string[]; rows: EvalCallRow[] }
|
||||
|
||||
/** The reference emitter a consuming repo copies: binding lines, preconditions, one line per call purpose, the end line. */
|
||||
export function formatEvalCalls(p: EvalCallPlan): string {
|
||||
return [
|
||||
'# gstack eval call plan, wire format v1 (docs/eval-calls.md); priced by `gstack-eval-plan price-gate`, never executed',
|
||||
`GSTACK_EVAL_GATE v1: ${p.gate}`,
|
||||
`GSTACK_EVAL_HEAD v1: ${p.head}`,
|
||||
...p.inputs.map(i => `GSTACK_EVAL_INPUT v1: sha256=${i.sha256} path=${i.path}`),
|
||||
...p.preconditions.map(t => `GSTACK_EVAL_PRECONDITION: ${t}`),
|
||||
...p.rows.map(r => `GSTACK_EVAL_CALLS v1: ${EVAL_CALL_KEYS.map(k => `${k}=${r[k]}`).join(' ')}`),
|
||||
`GSTACK_EVAL_CALLS v1 end: lines=${p.rows.length}`,
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
|
||||
const INT_KEYS = ['calls', 'in_tokens', 'out_tokens', 'max_out_tokens'] as const;
|
||||
|
||||
function parseCallRow(body: string, at: string): EvalCallRow | string {
|
||||
const fields: Record<string, string> = {};
|
||||
for (const tok of body.trim().split(/ +/)) {
|
||||
const eq = tok.indexOf('=');
|
||||
if (eq <= 0) return `${at}: ${JSON.stringify(tok)} is not key=value`;
|
||||
const key = tok.slice(0, eq);
|
||||
if (!(EVAL_CALL_KEYS as readonly string[]).includes(key)) return `${at}: unknown key ${key}`;
|
||||
if (key in fields) return `${at}: duplicate key ${key}`;
|
||||
fields[key] = tok.slice(eq + 1);
|
||||
}
|
||||
for (const key of EVAL_CALL_KEYS) if (!(key in fields)) return `${at}: missing key ${key}`;
|
||||
for (const key of INT_KEYS) if (!/^\d+$/.test(fields[key]!)) return `${at}: ${key} must be a non-negative integer (got ${JSON.stringify(fields[key])})`;
|
||||
if (Number(fields.calls) < 1) return `${at}: calls must be at least 1`;
|
||||
if (fields.scope !== 'full' && fields.scope !== 'smoke') return `${at}: scope must be full or smoke (got ${JSON.stringify(fields.scope)})`;
|
||||
if (!/^[A-Za-z0-9._:-]+$/.test(fields.purpose!)) return `${at}: purpose must match [A-Za-z0-9._:-]+`;
|
||||
if (!fields.model) return `${at}: model is empty`;
|
||||
return { purpose: fields.purpose!, model: fields.model, calls: Number(fields.calls), in_tokens: Number(fields.in_tokens), out_tokens: Number(fields.out_tokens), max_out_tokens: Number(fields.max_out_tokens), scope: fields.scope };
|
||||
}
|
||||
|
||||
/** Strict v1 parser: a malformed line, an unknown kind or version, a missing binding or a missing or miscounted end line refuses, naming the line. */
|
||||
export function parseEvalCalls(text: string): { plan: EvalCallPlan } | { error: string } {
|
||||
const plan: EvalCallPlan = { gate: '', head: '', inputs: [], preconditions: [], rows: [] };
|
||||
const seen = new Set<string>();
|
||||
let gate: string | null = null; let head: string | null = null; let ended = false;
|
||||
const lines = text.split('\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const raw = lines[i]!.replace(/\r$/, ''); const at = `line ${i + 1}`;
|
||||
if (!raw.trim() || raw.startsWith('#')) continue;
|
||||
if (ended) return { error: `${at}: content after the end line` };
|
||||
const m = /^(GSTACK_EVAL_[A-Z_]+?)(?: (v\d+))?( end)?: (.*)$/.exec(raw);
|
||||
if (!m) return { error: `${at}: not a GSTACK_EVAL_ line (${JSON.stringify(raw.slice(0, 80))})` };
|
||||
const [, kind, version, end, body] = m as unknown as [string, string, string | undefined, string | undefined, string];
|
||||
if (kind === 'GSTACK_EVAL_PRECONDITION' && !version && !end) { plan.preconditions.push(body.trim()); continue; }
|
||||
if (!['GSTACK_EVAL_GATE', 'GSTACK_EVAL_HEAD', 'GSTACK_EVAL_INPUT', 'GSTACK_EVAL_CALLS'].includes(kind)) return { error: `${at}: unknown line kind ${kind}` };
|
||||
if (version !== 'v1') return { error: `${at}: ${kind} version ${version ?? '(none)'} is not supported (this gstack reads v1)` };
|
||||
if (end) {
|
||||
const n = kind === 'GSTACK_EVAL_CALLS' ? /^lines=(\d+)$/.exec(body.trim()) : null;
|
||||
if (!n) return { error: `${at}: malformed end line (expected GSTACK_EVAL_CALLS v1 end: lines=<n>)` };
|
||||
if (Number(n[1]) !== plan.rows.length) return { error: `${at}: end line says lines=${n[1]} but the file has ${plan.rows.length} call lines` };
|
||||
ended = true; continue;
|
||||
}
|
||||
if (kind === 'GSTACK_EVAL_GATE') {
|
||||
if (gate !== null) return { error: `${at}: second GSTACK_EVAL_GATE line` };
|
||||
if (!body.trim()) return { error: `${at}: empty gate command` };
|
||||
gate = body; continue;
|
||||
}
|
||||
if (kind === 'GSTACK_EVAL_HEAD') {
|
||||
if (head !== null) return { error: `${at}: second GSTACK_EVAL_HEAD line` };
|
||||
if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/.test(body.trim())) return { error: `${at}: head must be a full commit sha` };
|
||||
head = body.trim(); continue;
|
||||
}
|
||||
if (kind === 'GSTACK_EVAL_INPUT') {
|
||||
const inp = /^sha256=([0-9a-f]{64}) path=(.+)$/.exec(body);
|
||||
if (!inp) return { error: `${at}: expected GSTACK_EVAL_INPUT v1: sha256=<64 hex> path=<repo-relative path>` };
|
||||
if (path.isAbsolute(inp[2]!) || inp[2]!.split(/[\\/]/).includes('..')) return { error: `${at}: input path must be repo-relative without .. (${inp[2]})` };
|
||||
plan.inputs.push({ path: inp[2]!, sha256: inp[1]! }); continue;
|
||||
}
|
||||
const row = parseCallRow(body, at);
|
||||
if (typeof row === 'string') return { error: row };
|
||||
if (seen.has(`${row.purpose}\0${row.model}`)) return { error: `${at}: duplicate purpose=${row.purpose} model=${row.model}` };
|
||||
seen.add(`${row.purpose}\0${row.model}`);
|
||||
plan.rows.push(row);
|
||||
}
|
||||
if (gate === null) return { error: 'missing GSTACK_EVAL_GATE v1 line (the exact gate command this plan prices)' };
|
||||
if (head === null) return { error: 'missing GSTACK_EVAL_HEAD v1 line (the commit the plan was produced at)' };
|
||||
if (!ended) return { error: 'missing end line (GSTACK_EVAL_CALLS v1 end: lines=<n>); a partial file is refused' };
|
||||
return { plan: { ...plan, gate, head } };
|
||||
}
|
||||
|
||||
export interface PricedCallRow extends EvalCallRow { usd: number | null; reservation_usd: number | null }
|
||||
export interface GateReceipt {
|
||||
schema_version: 1; kind: 'gstack-eval-price-gate'; created_at: string; gstack_revision: string;
|
||||
gate_command: string; gate_sha256: string; head_sha: string;
|
||||
plan: { file: string | null; source: 'flag' | 'price_file' | 'none'; sha256: string | null; declared_head: string | null };
|
||||
inputs: EvalPlanInput[]; preconditions: string[]; rows: PricedCallRow[];
|
||||
total_usd: number | null; calls: number; purposes: string[]; cap_usd: number | null; smoke_cap_usd: number | null;
|
||||
result: 'priced' | 'price_missing' | 'over_cap' | 'unpriced' | 'unpriced_override'; code: ResultCodeName | null;
|
||||
unpriced_reason: string | null; unpriced_override: string | null; excluded_billing: readonly string[];
|
||||
}
|
||||
export interface PlanSource { text: Buffer | null; file: string | null; source: GateReceipt['plan']['source']; repoPath: string | null; why: string | null }
|
||||
export interface PriceGateOptions { repoRoot: string; gate: string; plan: PlanSource; capUsd?: number; smokeCapUsd?: number; allowUnpriced?: string; now?: Date }
|
||||
export interface PriceGateResult { lines: string[]; receipt: GateReceipt | null; error?: { code: ResultCodeName; message: string } }
|
||||
|
||||
const sha256 = (b: Buffer | string) => createHash('sha256').update(b).digest('hex');
|
||||
function gitOut(repoRoot: string, args: string[]): Buffer | null {
|
||||
const r = spawnSync('git', args, { cwd: repoRoot, timeout: 30_000 });
|
||||
return r.status === 0 ? r.stdout : null;
|
||||
}
|
||||
|
||||
export interface PlanSourceRefusal { code: ResultCodeName; message: string; refused: boolean }
|
||||
|
||||
/**
|
||||
* `--plan-file` wins; else the base policy's `price_file`, read from the HEAD
|
||||
* commit's bytes. Only "no plan declared" is unpriced (overridable); a declared
|
||||
* file missing at HEAD is stale, an unresolvable base or an invalid policy refuses.
|
||||
*/
|
||||
export function resolvePlanSource(repoRoot: string, o: { planFile?: string; base: string }): PlanSource | PlanSourceRefusal {
|
||||
if (o.planFile) {
|
||||
const file = path.resolve(o.planFile);
|
||||
const rel = path.relative(repoRoot, file);
|
||||
return { text: fs.readFileSync(file), file, source: 'flag', repoPath: rel && !rel.startsWith('..') && !path.isAbsolute(rel) ? rel.split(path.sep).join('/') : null, why: null };
|
||||
}
|
||||
const load = loadConfigFromBase(repoRoot, o.base);
|
||||
if (isConfigUnavailable(load)) return { code: 'POLICY_SOURCE_UNAVAILABLE', message: `no --plan-file and the base policy cannot be read (${load.detail})`, refused: true };
|
||||
if (load.errors.length) return { code: 'PREGATE_CONFIG_INVALID', message: `${load.label}: ${load.errors.map(e => `${e.path}: ${e.message}`).join('; ')}`, refused: true };
|
||||
const rel = load.config.price_file;
|
||||
if (!rel) return { text: null, file: null, source: 'none', repoPath: null, why: `no --plan-file and ${load.label} declares no price_file` };
|
||||
const text = gitOut(repoRoot, ['show', `HEAD:${rel}`]);
|
||||
if (!text) return { code: 'EVAL_PLAN_STALE', message: `price_file ${rel} (declared in ${load.label}) is not in the HEAD commit`, refused: false };
|
||||
return { text, file: rel, source: 'price_file', repoPath: rel, why: null };
|
||||
}
|
||||
|
||||
/** Stale binding: the gate command, the head (only the plan file itself may change after it) and every input hash at HEAD. */
|
||||
function checkBinding(o: PriceGateOptions, plan: EvalCallPlan, headSha: string): string | null {
|
||||
if (plan.gate !== o.gate) return `gate command differs: --gate ${JSON.stringify(o.gate)} vs plan ${JSON.stringify(plan.gate)}`;
|
||||
if (plan.head !== headSha) {
|
||||
const diff = gitOut(o.repoRoot, ['diff', '--name-only', plan.head, headSha]);
|
||||
if (diff === null) return `declared head ${plan.head.slice(0, 12)} is not a commit in this repository (HEAD ${headSha.slice(0, 12)})`;
|
||||
const changed = diff.toString('utf8').split('\n').filter(Boolean);
|
||||
if (changed.some(f => f !== o.plan.repoPath)) return `declared head ${plan.head.slice(0, 12)} is behind HEAD ${headSha.slice(0, 12)}: ${changed.length} file(s) changed since (${changed.slice(0, 3).join(', ')})`;
|
||||
}
|
||||
for (const input of plan.inputs) {
|
||||
const bytes = gitOut(o.repoRoot, ['show', `HEAD:${input.path}`]);
|
||||
if (!bytes) return `input ${input.path} is not in the HEAD commit`;
|
||||
const got = sha256(bytes);
|
||||
if (got !== input.sha256) return `input ${input.path} sha256 ${got.slice(0, 12)} at HEAD != declared ${input.sha256.slice(0, 12)}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function gstackRevision(): string {
|
||||
const root = path.resolve(import.meta.dir, '..');
|
||||
const version = fs.existsSync(path.join(root, 'VERSION')) ? fs.readFileSync(path.join(root, 'VERSION'), 'utf8').trim() : 'unknown';
|
||||
const sha = gitOut(root, ['rev-parse', '--short', 'HEAD'])?.toString('utf8').trim();
|
||||
return sha ? `${version} (${sha})` : version;
|
||||
}
|
||||
|
||||
const amount = (v: number | null) => (v === null ? 'unknown' : String(+v.toFixed(6)));
|
||||
|
||||
function priceRows(rows: EvalCallRow[]): PricedCallRow[] {
|
||||
return rows.map(r => {
|
||||
if (!hasPricing(r.model)) return { ...r, usd: null, reservation_usd: null };
|
||||
return { ...r, usd: estimateCostUsd({ input: r.in_tokens, output: r.out_tokens }, r.model), reservation_usd: estimateCostUsd({ input: Math.ceil(r.in_tokens / r.calls), output: r.max_out_tokens }, r.model) };
|
||||
});
|
||||
}
|
||||
|
||||
/** Price a paid gate from its call-plan file. Malformed or stale files return an error and no receipt; every other outcome returns a receipt. */
|
||||
export function priceGate(o: PriceGateOptions): PriceGateResult {
|
||||
const headSha = gitOut(o.repoRoot, ['rev-parse', 'HEAD'])?.toString('utf8').trim() ?? '';
|
||||
let plan: EvalCallPlan | null = null;
|
||||
if (o.plan.text) {
|
||||
const parsed = parseEvalCalls(o.plan.text.toString('utf8'));
|
||||
if ('error' in parsed) return { lines: [], receipt: null, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.plan.file}: ${parsed.error}` } };
|
||||
const stale = checkBinding(o, parsed.plan, headSha);
|
||||
if (stale) return { lines: [], receipt: null, error: { code: 'EVAL_PLAN_STALE', message: `${o.plan.file}: ${stale}` } };
|
||||
plan = parsed.plan;
|
||||
}
|
||||
const rows = priceRows(plan?.rows ?? []);
|
||||
const total = rows.some(r => r.usd === null) ? null : +rows.reduce((s, r) => s + r.usd!, 0).toFixed(6);
|
||||
const lines = [
|
||||
...(plan ? [`GATE_PLAN: file=${o.plan.file} source=${o.plan.source} sha256=${sha256(o.plan.text!)} head=${plan.head.slice(0, 12)}${plan.head === headSha ? '' : ` (HEAD ${headSha.slice(0, 12)}; only the plan file changed since)`} inputs=${plan.inputs.length} verified`] : []),
|
||||
...(plan?.preconditions ?? []).map(t => `GATE_PRECONDITION: ${t}`),
|
||||
...rows.map(r => `GATE_CALLS: purpose=${r.purpose} model=${r.model} calls=${r.calls} usd=${amount(r.usd)} reservation_usd=${amount(r.reservation_usd)} scope=${r.scope}`),
|
||||
];
|
||||
let error: PriceGateResult['error'];
|
||||
let result: GateReceipt['result'] = 'priced';
|
||||
const unpricedWhy = !plan ? o.plan.why ?? 'no call-plan file' : rows.length === 0 ? `${o.plan.file} has no GSTACK_EVAL_CALLS lines` : null;
|
||||
if (unpricedWhy) {
|
||||
result = o.allowUnpriced ? 'unpriced_override' : 'unpriced';
|
||||
if (o.allowUnpriced) lines.push(`GATE_UNPRICED_OVERRIDE: ${o.allowUnpriced} (owner-authorized override; not authorization to spend)`);
|
||||
else error = { code: 'EVAL_GATE_UNPRICED', message: unpricedWhy };
|
||||
} else if (total === null) {
|
||||
result = 'price_missing';
|
||||
error = { code: 'EVAL_PRICE_MISSING', message: `no price row for ${rows.filter(r => r.usd === null).map(r => r.model).join(', ')}; the total is unknown, never zero` };
|
||||
} else {
|
||||
const over: string[] = [];
|
||||
if (o.capUsd !== undefined || o.smokeCapUsd !== undefined) {
|
||||
for (const r of rows) {
|
||||
const smoke = r.scope === 'smoke' && o.smokeCapUsd !== undefined && r.reservation_usd! > o.smokeCapUsd;
|
||||
const full = o.capUsd !== undefined && r.reservation_usd! > o.capUsd;
|
||||
if (!smoke && !full) continue;
|
||||
const why = `$${r.reservation_usd!.toFixed(4)} > ${smoke ? `smoke cap $${o.smokeCapUsd!.toFixed(2)}` : `cap $${o.capUsd!.toFixed(2)}`}`;
|
||||
lines.push(`CHECK: reservation-fits-cap exceeds (${why}) purpose=${r.purpose}`);
|
||||
over.push(`reservation-fits-cap: purpose=${r.purpose} one call ${why}`);
|
||||
}
|
||||
if (!over.length) lines.push(`CHECK: reservation-fits-cap ok (largest one-call reservation $${Math.max(0, ...rows.map(r => r.reservation_usd!)).toFixed(4)})`);
|
||||
}
|
||||
if (o.capUsd !== undefined && total > o.capUsd) over.unshift(`total $${total.toFixed(4)} > cap $${o.capUsd.toFixed(2)}`);
|
||||
if (over.length) { result = 'over_cap'; error = { code: 'EVAL_GATE_OVER_CAP', message: over.join('; ') }; }
|
||||
}
|
||||
const receipt: GateReceipt = {
|
||||
schema_version: 1, kind: 'gstack-eval-price-gate', created_at: (o.now ?? new Date()).toISOString(), gstack_revision: gstackRevision(),
|
||||
gate_command: o.gate, gate_sha256: sha256(o.gate), head_sha: headSha,
|
||||
plan: { file: o.plan.file, source: o.plan.source, sha256: o.plan.text ? sha256(o.plan.text) : null, declared_head: plan?.head ?? null },
|
||||
inputs: plan?.inputs ?? [], preconditions: plan?.preconditions ?? [], rows,
|
||||
total_usd: unpricedWhy ? null : total, calls: rows.reduce((s, r) => s + r.calls, 0), purposes: rows.map(r => r.purpose),
|
||||
cap_usd: o.capUsd ?? null, smoke_cap_usd: o.smokeCapUsd ?? null, result, code: error?.code ?? (unpricedWhy ? 'EVAL_GATE_UNPRICED' : null),
|
||||
unpriced_reason: unpricedWhy, unpriced_override: unpricedWhy ? o.allowUnpriced ?? null : null, excluded_billing: EXCLUDED_BILLING_CATEGORIES,
|
||||
};
|
||||
return { lines, receipt, error };
|
||||
}
|
||||
|
||||
export function renderGatePrice(r: GateReceipt, receiptPath: string): string {
|
||||
return `GATE_PRICE: total=${amount(r.total_usd)} calls=${r.calls} purposes=${r.purposes.join(',') || 'none'} receipt=${receiptPath}`;
|
||||
}
|
||||
+40
-4
@@ -20,7 +20,8 @@
|
||||
* start a priced run on such a model (bin/gstack-eval-plan dry-run).
|
||||
* - Anthropic: https://www.anthropic.com/pricing#api
|
||||
* - OpenAI: https://openai.com/api/pricing/
|
||||
* - Google AI: https://ai.google.dev/pricing
|
||||
* - Google AI: https://ai.google.dev/gemini-api/docs/pricing
|
||||
* - Voyage AI (embeddings): https://docs.voyageai.com/docs/pricing
|
||||
*/
|
||||
|
||||
export interface ModelPricing {
|
||||
@@ -45,7 +46,10 @@ export const PRICING: Record<string, ModelPricing> = {
|
||||
'claude-fable-5-1': { input_per_mtok: 10.00, output_per_mtok: 50.00, cached_input_per_mtok: 0.25, as_of: '2026-09' },
|
||||
// https://platform.claude.com/docs/en/models/opus-5-5/overview#pricing (list prices effective 2026-10-07)
|
||||
'claude-opus-5-5': { input_per_mtok: 4.00, output_per_mtok: 20.00, cached_input_per_mtok: 0.20, as_of: '2026-10' },
|
||||
'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' },
|
||||
// https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12)
|
||||
'claude-opus-4-7': { input_per_mtok: 5.00, output_per_mtok: 25.00, cached_input_per_mtok: 0.50, as_of: '2026-10' },
|
||||
// https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-10-12)
|
||||
'claude-sonnet-5-5': { input_per_mtok: 2.00, output_per_mtok: 10.00, cached_input_per_mtok: 0.10, as_of: '2026-10' },
|
||||
'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' },
|
||||
'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
|
||||
@@ -60,14 +64,46 @@ export const PRICING: Record<string, ModelPricing> = {
|
||||
'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' },
|
||||
|
||||
// Google
|
||||
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; standard tier through 2026-12-31,
|
||||
// the page lists $1.50 / $7.50 / $0.15 cached starting 2027-01-01)
|
||||
'gemini-3.8-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' },
|
||||
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; same schedule as gemini-3.8-flash)
|
||||
'gemini-3.6-flash': { input_per_mtok: 0.75, output_per_mtok: 3.75, cached_input_per_mtok: 0.075, as_of: '2026-10' },
|
||||
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; prompts <= 200k tokens, the long-context rate is excluded)
|
||||
'gemini-3.1-pro-preview': { input_per_mtok: 2.00, output_per_mtok: 12.00, cached_input_per_mtok: 0.20, as_of: '2026-10' },
|
||||
'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' },
|
||||
};
|
||||
|
||||
/** A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`) resolves to its table row. */
|
||||
/**
|
||||
* Embedding models bill input tokens only (output_per_mtok 0), so they live
|
||||
* beside PRICING rather than in it: eval arms never resolve to an embedding
|
||||
* model, but a call plan (docs/eval-calls.md) prices gbrain's embedding calls.
|
||||
*/
|
||||
export const EMBEDDING_PRICING: Record<string, ModelPricing> = {
|
||||
// https://docs.voyageai.com/docs/pricing (fetched 2026-10-12; list price after the free-token allowance)
|
||||
'voyage-4': { input_per_mtok: 0.06, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-4-large': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-4-lite': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-code-4': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-code-3': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-3-large': { input_per_mtok: 0.18, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'voyage-multimodal-3': { input_per_mtok: 0.12, output_per_mtok: 0, as_of: '2026-10' }, // text tokens; pixels excluded
|
||||
// https://developers.openai.com/api/docs/pricing (fetched 2026-10-12)
|
||||
'text-embedding-3-large': { input_per_mtok: 0.13, output_per_mtok: 0, as_of: '2026-10' },
|
||||
'text-embedding-3-small': { input_per_mtok: 0.02, output_per_mtok: 0, as_of: '2026-10' },
|
||||
// https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-10-12; text input, standard tier)
|
||||
'gemini-embedding-2': { input_per_mtok: 0.20, output_per_mtok: 0, as_of: '2026-10' },
|
||||
};
|
||||
|
||||
/**
|
||||
* A provider-prefixed id (`openai/gpt-6-astra`, `anthropic/claude-fable-5-1`,
|
||||
* or gbrain's `voyage:voyage-4`) resolves to its table row.
|
||||
*/
|
||||
export function pricingRow(model: string | undefined): ModelPricing | undefined {
|
||||
if (!model) return undefined;
|
||||
return PRICING[model] ?? PRICING[model.replace(/^[a-z-]+\//, '')];
|
||||
const bare = model.replace(/^[a-z-]+[/:]/, '');
|
||||
return PRICING[model] ?? PRICING[bare] ?? EMBEDDING_PRICING[model] ?? EMBEDDING_PRICING[bare];
|
||||
}
|
||||
|
||||
export function hasPricing(model: string | undefined): boolean {
|
||||
|
||||
@@ -476,6 +476,21 @@ export const RESULT_CODES = {
|
||||
summary: 'a push or PR would publish sealed or eval output (a sealed path, a transcript, a quoted answer, a seed or a local home path) to a public or unknown destination, or the sealed policy is unreadable',
|
||||
fix: 'remove it from every published commit, or add one reviewed `rule<TAB>path<TAB>sha<TAB>reason` line to the base branch\'s .gstack/sealed-allow.tsv; a private destination sets sealed.visibility "private" in the base branch\'s .gstack/pregate.json',
|
||||
},
|
||||
EVAL_GATE_UNPRICED: {
|
||||
anchor: 'eval-gate-unpriced',
|
||||
summary: 'the paid gate has no call-plan file (no `--plan-file`, no base-policy `price_file`) or the file has no GSTACK_EVAL_CALLS lines, so its cost is unknown',
|
||||
fix: 'have the repo commit a call-plan file (docs/eval-calls.md) and set `price_file` in .gstack/pregate.json, or pass `--allow-unpriced "<reason>"` only on the owner\'s say-so',
|
||||
},
|
||||
EVAL_GATE_OVER_CAP: {
|
||||
anchor: 'eval-gate-over-cap',
|
||||
summary: 'the paid gate\'s priced total, or one call\'s reservation (input plus maximum output), is above the cap',
|
||||
fix: 'cut questions, arms or `max_out_tokens` in the call plan, or raise `--cap-usd` / `--smoke-cap-usd` deliberately with the owner; never split calls to slip under it',
|
||||
},
|
||||
EVAL_PLAN_STALE: {
|
||||
anchor: 'eval-plan-stale',
|
||||
summary: 'the call-plan file\'s binding does not match (a different gate command, a head behind HEAD, an input whose sha256 at HEAD differs) or the declared price_file is not in HEAD',
|
||||
fix: 'regenerate the call-plan file at HEAD for the exact gate command you are approving, commit it, and rerun `gstack-eval-plan price-gate`',
|
||||
},
|
||||
} as const satisfies Record<string, ResultCode>;
|
||||
|
||||
export type ResultCodeName = keyof typeof RESULT_CODES;
|
||||
|
||||
@@ -23,10 +23,10 @@ test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
|
||||
});
|
||||
|
||||
test('estimateCostUsd computes correctly for known Claude model', () => {
|
||||
// claude-opus-4-7: $15/MTok input, $75/MTok output
|
||||
// 1M input + 0.5M output = $15 + $37.50 = $52.50
|
||||
// claude-opus-4-7: $5/MTok input, $25/MTok output (platform.claude.com pricing, fetched 2026-10-12)
|
||||
// 1M input + 0.5M output = $5 + $12.50 = $17.50
|
||||
const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7');
|
||||
expect(cost).toBeCloseTo(52.50, 2);
|
||||
expect(cost).toBeCloseTo(17.50, 2);
|
||||
});
|
||||
|
||||
for (const model of [CLAUDE_FRONTIER_EVAL_MODEL, CODEX_FRONTIER_MODEL]) {
|
||||
@@ -48,12 +48,12 @@ test('Astra cache reads use its standard cached-input rate', () => {
|
||||
|
||||
test('estimateCostUsd applies cached input discount alongside uncached input', () => {
|
||||
// tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%.
|
||||
// 0 uncached input, 1M cached → 10% of 15 = $1.50
|
||||
// 0 uncached input, 1M cached → 10% of 5 = $0.50
|
||||
const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7');
|
||||
expect(cost1).toBeCloseTo(1.50, 2);
|
||||
// 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25
|
||||
expect(cost1).toBeCloseTo(0.50, 2);
|
||||
// 500K uncached input + 500K cached → $2.50 + $0.25 = $2.75
|
||||
const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7');
|
||||
expect(cost2).toBeCloseTo(8.25, 2);
|
||||
expect(cost2).toBeCloseTo(2.75, 2);
|
||||
});
|
||||
|
||||
test('PRICING table covers the key model families', () => {
|
||||
|
||||
@@ -3,16 +3,19 @@
|
||||
* /eval-plan machinery in lib/eval-plan.ts: preregistration check, priced arms,
|
||||
* the $0 dry run, and the priced pilot ledger with its spend table.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { EXCLUDED_BILLING_CATEGORIES, PRICING, estimateCostUsd, hasPricing, pricingRow } from '../lib/pricing';
|
||||
import * as helperPricing from './helpers/pricing';
|
||||
import {
|
||||
PREREG_SECTIONS, checkPrereg, dryRun, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable,
|
||||
PREREG_SECTIONS, checkPrereg, dryRun, formatEvalCalls, parseEvalCalls, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable,
|
||||
resolveArms, settlePilot, startPilot,
|
||||
} from '../lib/eval-plan';
|
||||
import { capyHost, type CapyHost } from './helpers/capy-host';
|
||||
|
||||
const dirs: string[] = [];
|
||||
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-plan-')); dirs.push(d); return d; };
|
||||
@@ -156,3 +159,273 @@ describe('priced pilot', () => {
|
||||
expect(r.projection.verdict).toBe('unknown');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CD2 (field wave item 15): paid gates priced from a call-plan DATA file.
|
||||
// `price-gate` never executes anything: it validates the file's binding (gate
|
||||
// command, head, input hashes at HEAD) and prices the GSTACK_EVAL_CALLS v1 rows.
|
||||
// ---------------------------------------------------------------------------
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-eval-plan');
|
||||
|
||||
function scratchRepo(env: Record<string, string>): { repo: string; head: string; rowsSha: string } {
|
||||
const repo = path.join(env.TMPDIR!, 'repo');
|
||||
fs.mkdirSync(repo, { recursive: true });
|
||||
const git = (...args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8' });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
};
|
||||
git('init', '-q', '-b', 'main');
|
||||
fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n');
|
||||
git('add', 'rows.jsonl');
|
||||
git('commit', '-q', '-m', 'rows');
|
||||
return { repo, head: git('rev-parse', 'HEAD'), rowsSha: createHash('sha256').update(fs.readFileSync(path.join(repo, 'rows.jsonl'))).digest('hex') };
|
||||
}
|
||||
|
||||
const GATE = 'bun run eval:synopsis --questions 50 --arms opus,sol';
|
||||
const twoRows = [
|
||||
{ purpose: 'answer', model: 'anthropic/claude-opus-5-5', calls: 100, in_tokens: 400_000, out_tokens: 200_000, max_out_tokens: 8_192, scope: 'full' as const },
|
||||
{ purpose: 'judge', model: 'gpt-6.1-sol', calls: 100, in_tokens: 300_000, out_tokens: 50_000, max_out_tokens: 2_048, scope: 'full' as const },
|
||||
];
|
||||
|
||||
function runGate(host: CapyHost, cwd: string, args: string[], extraEnv: Record<string, string> = {}) {
|
||||
const r = spawnSync(BIN, ['price-gate', ...args], { cwd, env: { ...host.env, ...extraEnv }, encoding: 'utf8' });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
}
|
||||
|
||||
describe('price-gate: the call-plan file format (formatEvalCalls / parseEvalCalls)', () => {
|
||||
test('the reference emitter round-trips: binding lines, preconditions, call lines and the end line', () => {
|
||||
const plan = { gate: GATE, head: 'a'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: 'b'.repeat(64) }], preconditions: ['ANTHROPIC_API_KEY set at run time'], rows: twoRows };
|
||||
const text = formatEvalCalls(plan);
|
||||
expect(text).toContain(`GSTACK_EVAL_GATE v1: ${GATE}\n`);
|
||||
expect(text).toContain('GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full\n');
|
||||
expect(text.trimEnd().split('\n').at(-1)).toBe('GSTACK_EVAL_CALLS v1 end: lines=2');
|
||||
const parsed = parseEvalCalls(text);
|
||||
expect('plan' in parsed && parsed.plan).toEqual(plan);
|
||||
});
|
||||
|
||||
test('a malformed, partial or unknown-version file refuses and names the line', () => {
|
||||
const good = formatEvalCalls({ gate: GATE, head: 'a'.repeat(40), inputs: [], preconditions: [], rows: twoRows });
|
||||
const err = (text: string) => { const r = parseEvalCalls(text); return 'error' in r ? r.error : ''; };
|
||||
expect(err(good.replace(/\nGSTACK_EVAL_CALLS v1 end: lines=2\n$/, '\n'))).toMatch(/end line/);
|
||||
expect(err(good.replace('end: lines=2', 'end: lines=3'))).toMatch(/lines=3.*2 call lines/);
|
||||
expect(err(good.replace('calls=100 in_tokens=400000', 'calls=1e2 in_tokens=400000'))).toMatch(/line 4.*calls/);
|
||||
expect(err(good.replace('scope=full\nGSTACK_EVAL_CALLS v1: purpose=judge', 'scope=full extra=1\nGSTACK_EVAL_CALLS v1: purpose=judge'))).toMatch(/line 4.*extra/);
|
||||
expect(err(good.replace(' max_out_tokens=8192', ''))).toMatch(/line 4.*max_out_tokens/);
|
||||
expect(err(good.replace('GSTACK_EVAL_CALLS v1: purpose=judge', 'GSTACK_EVAL_CALLS v2: purpose=judge'))).toMatch(/line 5.*version/);
|
||||
expect(err(good + 'GSTACK_EVAL_PRECONDITION: after the end\n')).toMatch(/after the end line/);
|
||||
expect(err(good.replace(/^GSTACK_EVAL_HEAD.*\n/m, ''))).toMatch(/GSTACK_EVAL_HEAD/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('price-gate: binding, pricing and the receipt (bin, simulated Capy host)', () => {
|
||||
let host: CapyHost;
|
||||
beforeEach(() => { host = capyHost(); });
|
||||
afterEach(() => host.cleanup());
|
||||
|
||||
function writePlan(repo: string, plan: Parameters<typeof formatEvalCalls>[0]): string {
|
||||
const file = path.join(host.base, 'plan.calls');
|
||||
fs.writeFileSync(file, formatEvalCalls(plan));
|
||||
return file;
|
||||
}
|
||||
|
||||
test('two call lines and one precondition are priced and listed, and the receipt binds price to gate, head, inputs and plan bytes', () => {
|
||||
const { repo, head, rowsSha } = scratchRepo(host.env);
|
||||
const file = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: ['ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time'], rows: twoRows });
|
||||
const out = path.join(host.base, 'receipts');
|
||||
const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '20', '--out', out]);
|
||||
expect(r.err).toBe('');
|
||||
expect(r.code).toBe(0);
|
||||
// opus-5-5: 0.4 MTok × $4 + 0.2 MTok × $20 = $5.60; sol: 0.3 × $2 + 0.05 × $10 = $1.10
|
||||
expect(r.out).toContain('GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full');
|
||||
expect(r.out).toContain('GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full');
|
||||
expect(r.out).toContain('GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time');
|
||||
const price = r.out.split('\n').find(l => l.startsWith('GATE_PRICE: '))!;
|
||||
expect(price).toMatch(/^GATE_PRICE: total=6\.7 calls=200 purposes=answer,judge receipt=\S+$/);
|
||||
const receiptPath = price.split('receipt=')[1]!;
|
||||
expect(path.dirname(receiptPath)).toBe(out);
|
||||
const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8'));
|
||||
expect(receipt).toMatchObject({
|
||||
schema_version: 1, result: 'priced', code: null, gate_command: GATE, gate_sha256: createHash('sha256').update(GATE).digest('hex'),
|
||||
head_sha: head, plan: { file, source: 'flag', declared_head: head, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') },
|
||||
inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], total_usd: 6.7, calls: 200, cap_usd: 20, unpriced_override: null,
|
||||
});
|
||||
expect(typeof receipt.gstack_revision).toBe('string');
|
||||
});
|
||||
|
||||
test('the base policy price_file is read from the HEAD commit; a plan committed on top of its own head stays fresh until another file changes', () => {
|
||||
const { repo, rowsSha } = scratchRepo(host.env);
|
||||
const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim();
|
||||
fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/gate.calls' }));
|
||||
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
|
||||
const produced = git('rev-parse', 'HEAD');
|
||||
git('update-ref', 'refs/remotes/origin/main', produced);
|
||||
fs.mkdirSync(path.join(repo, 'evals'));
|
||||
fs.writeFileSync(path.join(repo, 'evals', 'gate.calls'), formatEvalCalls({ gate: GATE, head: produced, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows }));
|
||||
git('add', 'evals/gate.calls'); git('commit', '-q', '-m', 'call plan');
|
||||
const out = path.join(host.base, 'receipts');
|
||||
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
|
||||
expect(r.err).toBe('');
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toContain(`GATE_PLAN: file=evals/gate.calls source=price_file`);
|
||||
expect(r.out).toContain('only the plan file changed since');
|
||||
const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
|
||||
expect(receipt).toMatchObject({ plan: { source: 'price_file', declared_head: produced }, head_sha: git('rev-parse', 'HEAD'), total_usd: 6.7 });
|
||||
fs.writeFileSync(path.join(repo, 'README.md'), 'later\n');
|
||||
git('add', 'README.md'); git('commit', '-q', '-m', 'later');
|
||||
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/behind HEAD.*README\.md.*\(EVAL_PLAN_STALE\)/);
|
||||
});
|
||||
|
||||
test('a declared price_file missing at HEAD refuses EVAL_PLAN_STALE (not overridable); an unresolvable base or an invalid pregate.json exits 3', () => {
|
||||
const { repo } = scratchRepo(host.env);
|
||||
const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8' }).stdout.trim();
|
||||
const out = path.join(host.base, 'receipts');
|
||||
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
|
||||
expect(r.code).toBe(3);
|
||||
expect(r.err).toContain('(POLICY_SOURCE_UNAVAILABLE)');
|
||||
fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/missing.calls' }));
|
||||
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
|
||||
git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
|
||||
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/evals\/missing\.calls.*\(EVAL_PLAN_STALE\)/);
|
||||
expect(r.out).not.toContain('GATE_UNPRICED_OVERRIDE');
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: '../outside.calls' }));
|
||||
git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'bad policy');
|
||||
git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
|
||||
r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
|
||||
expect(r.code).toBe(3);
|
||||
expect(r.err).toContain('(PREGATE_CONFIG_INVALID)');
|
||||
expect(fs.existsSync(out)).toBe(false);
|
||||
});
|
||||
|
||||
test('an unknown or misspelled flag exits 2 instead of pricing without it', () => {
|
||||
const { repo } = scratchRepo(host.env);
|
||||
expect(runGate(host, repo, ['--gate', GATE, '--cap', '5']).code).toBe(2);
|
||||
expect(runGate(host, repo, ['--cap-usd', '5']).code).toBe(2);
|
||||
});
|
||||
|
||||
test('nothing in the plan file is executed: a gate command and precondition that would create markers leave none', () => {
|
||||
const { repo, head, rowsSha } = scratchRepo(host.env);
|
||||
const marker = path.join(host.base, 'gate-ran');
|
||||
const gate = `touch ${marker}`;
|
||||
const file = writePlan(repo, { gate, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [`$(touch ${marker}-pre)`], rows: twoRows });
|
||||
const r = runGate(host, repo, ['--plan-file', file, '--gate', gate]);
|
||||
expect(r.code).toBe(0);
|
||||
expect(fs.existsSync(marker)).toBe(false);
|
||||
expect(fs.existsSync(`${marker}-pre`)).toBe(false);
|
||||
});
|
||||
|
||||
test('a stale head, an input-hash mismatch and a gate-command mismatch each refuse EVAL_PLAN_STALE with no receipt', () => {
|
||||
const { repo, head, rowsSha } = scratchRepo(host.env);
|
||||
const out = path.join(host.base, 'receipts');
|
||||
const staleHead = writePlan(repo, { gate: GATE, head: 'f'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
|
||||
let r = runGate(host, repo, ['--plan-file', staleHead, '--gate', GATE, '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/head .*ffffffffffff.*\(EVAL_PLAN_STALE\)/);
|
||||
const badInput = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: '0'.repeat(64) }], preconditions: [], rows: twoRows });
|
||||
r = runGate(host, repo, ['--plan-file', badInput, '--gate', GATE, '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/rows\.jsonl.*sha256.*\(EVAL_PLAN_STALE\)/);
|
||||
const good = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
|
||||
r = runGate(host, repo, ['--plan-file', good, '--gate', `${GATE} --questions 500`, '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/gate command.*\(EVAL_PLAN_STALE\)/);
|
||||
expect(fs.existsSync(out)).toBe(false);
|
||||
});
|
||||
|
||||
test('malformed, partial (no end line) and empty-of-binding files refuse EVAL_DRY_RUN_FAILED naming the line', () => {
|
||||
const { repo, head } = scratchRepo(host.env);
|
||||
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
|
||||
fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/GSTACK_EVAL_CALLS v1 end: lines=2\n$/, ''));
|
||||
let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/end line.*\(EVAL_DRY_RUN_FAILED\)/);
|
||||
fs.writeFileSync(file, formatEvalCalls({ gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }).replace('calls=100', 'calls=lots'));
|
||||
r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/line \d+.*calls.*\(EVAL_DRY_RUN_FAILED\)/);
|
||||
});
|
||||
|
||||
test('no plan file, or a file with no call lines, is EVAL_GATE_UNPRICED exit 1; --allow-unpriced exits 0 with a recorded override', () => {
|
||||
const { repo, head, rowsSha } = scratchRepo(host.env);
|
||||
spawnSync('git', ['update-ref', 'refs/remotes/origin/main', head], { cwd: repo, env: host.env });
|
||||
const out = path.join(host.base, 'receipts');
|
||||
let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
|
||||
const empty = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: [] });
|
||||
r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
|
||||
expect(r.out).toContain('GATE_PRICE: total=unknown calls=0 purposes=none');
|
||||
r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out, '--allow-unpriced', 'owner accepts a bounded smoke on a seat']);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toContain('GATE_UNPRICED_OVERRIDE: owner accepts a bounded smoke on a seat (owner-authorized override; not authorization to spend)');
|
||||
const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
|
||||
expect(receipt).toMatchObject({ result: 'unpriced_override', code: 'EVAL_GATE_UNPRICED', unpriced_override: 'owner accepts a bounded smoke on a seat', total_usd: null });
|
||||
});
|
||||
|
||||
test('an unpriced model makes its row and the total unknown, never zero, and fails EVAL_PRICE_MISSING', () => {
|
||||
const { repo, head } = scratchRepo(host.env);
|
||||
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [twoRows[0]!, { ...twoRows[1]!, model: 'acme/made-up-model' }] });
|
||||
const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('GATE_CALLS: purpose=judge model=acme/made-up-model calls=100 usd=unknown reservation_usd=unknown scope=full');
|
||||
expect(r.out).toMatch(/GATE_PRICE: total=unknown calls=200 /);
|
||||
expect(r.err).toContain('(EVAL_PRICE_MISSING)');
|
||||
});
|
||||
|
||||
test('a total above --cap-usd is EVAL_GATE_OVER_CAP; a single-call reservation above the cap or the smoke cap fails reservation-fits-cap', () => {
|
||||
const { repo, head } = scratchRepo(host.env);
|
||||
const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
|
||||
let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '5']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.err).toMatch(/6\.7.*cap \$5.*\(EVAL_GATE_OVER_CAP\)/);
|
||||
const smoke = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [{ purpose: 'smoke-answer', model: 'claude-opus-5-5', calls: 2, in_tokens: 20_000, out_tokens: 4_000, max_out_tokens: 64_000, scope: 'smoke' }] });
|
||||
// one call: 10,000 in × $4 + 64,000 out × $20 per MTok = $1.32, under the $10 cap but over the $1 smoke cap
|
||||
r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '10', '--smoke-cap-usd', '1']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > smoke cap $1.00) purpose=smoke-answer');
|
||||
expect(r.err).toContain('(EVAL_GATE_OVER_CAP)');
|
||||
r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '1.2']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > cap $1.20) purpose=smoke-answer');
|
||||
});
|
||||
});
|
||||
|
||||
describe('dry-run reservation-fits-cap', () => {
|
||||
test('with --cap, one call per arm (largest row input + output budget), priced, must fit under the cap', () => {
|
||||
const d = tmp();
|
||||
const rows = path.join(d, 'rows.jsonl');
|
||||
fs.writeFileSync(rows, [JSON.stringify({ prompt: 'x'.repeat(4_000) }), JSON.stringify({ prompt: 'x'.repeat(40_000) })].join('\n') + '\n');
|
||||
const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
|
||||
// largest row 10,000 tokens; one gpt-6-astra call = 10,000 × $10 + 8,192 × $50 per MTok = $0.5096
|
||||
const tight = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 0.5 });
|
||||
expect(tight.checks.reservation_fits_cap).toMatchObject({ ok: false, cap: 0.5, max_input_tokens: 10_000 });
|
||||
expect(tight.checks.reservation_fits_cap!.per_arm['gpt-6-astra']).toBeCloseTo(0.5096, 6);
|
||||
expect(tight.pass).toBe(false);
|
||||
expect(renderDryRun(tight)).toContain('CHECK: reservation-fits-cap exceeds ($0.5096 > cap $0.50)');
|
||||
const roomy = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 1 });
|
||||
expect(roomy.checks.reservation_fits_cap!.ok).toBe(true);
|
||||
expect(roomy.pass).toBe(true);
|
||||
expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry }).checks.reservation_fits_cap).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pricing rows with fetched sources (2026-10-12)', () => {
|
||||
test('the newest Sonnet, the newest Gemini Flash and gbrain embedding models are priced; Opus 4.7 is at its current list price', () => {
|
||||
expect(hasPricing('anthropic/claude-sonnet-5-5')).toBe(true);
|
||||
expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-sonnet-5-5')).toBe(12);
|
||||
expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-opus-4-7')).toBe(30);
|
||||
expect(hasPricing('google/gemini-3.8-flash')).toBe(true);
|
||||
for (const m of ['voyage:voyage-4', 'openai:text-embedding-3-large', 'text-embedding-3-small', 'gemini-embedding-2']) expect(hasPricing(m)).toBe(true);
|
||||
expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'voyage:voyage-4')).toBe(0.06);
|
||||
expect(hasPricing('voyage-4-nano')).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user