From 3c9eeca1157d692225a933cbfd4737d9939edefc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 19:36:37 +0000 Subject: [PATCH] fix: a seat-metered arm that settles a charge is projected as a priced arm Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-eval-plan | 2 +- changelog.d/gst-566-eval-plan-seat-metered.md | 2 +- eval-plan/SKILL.md | 2 +- eval-plan/SKILL.md.tmpl | 2 +- lib/eval-plan.ts | 3 ++- test/pricing-eval-plan.test.ts | 6 +++++- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index f19260885..d4e8531a6 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -52,7 +52,7 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri and with --cap reservation-fits-cap (one call: largest row + output budget, priced) pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] [--seat-metered arm,...] --seat-metered: arms on a subscription with no incremental charge; they reserve $0, - settle only unknown, and sit outside the cap and the projection + settle unknown, and sit outside the cap and the projection until one settles a charge pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + per-arm extrapolation with a 95% band, then the approval ask price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] diff --git a/changelog.d/gst-566-eval-plan-seat-metered.md b/changelog.d/gst-566-eval-plan-seat-metered.md index 9a6b2648d..130828d29 100644 --- a/changelog.d/gst-566-eval-plan-seat-metered.md +++ b/changelog.d/gst-566-eval-plan-seat-metered.md @@ -1,2 +1,2 @@ ### Fixed -- **`/eval-plan` pilots with a subscription arm can now be approved, and the cost projection is per arm (#3653).** `gstack-eval-plan pilot start --seat-metered ` declares an arm that runs on a subscription with no extra charge (for example `claude -p` on a seat). Its attempts hold none of the spend cap, settle as `unknown` (a real charge still counts as spent), and get their own `PILOT_SEAT_METERED:` line, so the verdict covers the arms that cost money. `pilot report` now projects each arm from its own attempts (one `PILOT_ARM:` line each) and adds them up. Before, it averaged all arms together: a priced arm's cost was multiplied across arms with no price, and two arms at different prices got a band so wide that a pilot that fit the cap was told it exceeded it. The skill also says to stop a seat-metered run at its first quota or rate-limit error, with no retry (idea from #3273 by @merit-blake). +- **`/eval-plan` pilots with a subscription arm can now be approved, and the cost projection is per arm (#3653).** `gstack-eval-plan pilot start --seat-metered ` declares an arm that runs on a subscription with no extra charge (for example `claude -p` on a seat). Its attempts hold none of the spend cap, settle as `unknown` (an arm that settles a real charge is projected like any priced arm), and get their own `PILOT_SEAT_METERED:` line, so the verdict covers the arms that cost money. `pilot report` now projects each arm from its own attempts (one `PILOT_ARM:` line each) and adds them up. Before, it averaged all arms together: a priced arm's cost was multiplied across arms with no price, and two arms at different prices got a band so wide that a pilot that fit the cap was told it exceeded it. The skill also says to stop a seat-metered run at its first quota or rate-limit error, with no retry (idea from #3273 by @merit-blake). diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 5ebf340b7..ee298897c 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -259,7 +259,7 @@ eval command, then settle it with the actual cost from the provider's usage fiel An arm that runs on a subscription with no incremental charge (for example `claude -p` on a seat) is declared when the pilot starts: `--seat-metered `, each one also in `--arms`. Its attempts reserve $0, settle as `unknown` (a charge that does show up settles as -its number and counts as spent), and are listed on their own +its number, and the arm is then projected as a priced arm), and are listed on their own `PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms. The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start no new attempt, and release the attempts that never ran. An `unknown` settlement on an diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 19e930f5c..9a15b9134 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -181,7 +181,7 @@ eval command, then settle it with the actual cost from the provider's usage fiel An arm that runs on a subscription with no incremental charge (for example `claude -p` on a seat) is declared when the pilot starts: `--seat-metered `, each one also in `--arms`. Its attempts reserve $0, settle as `unknown` (a charge that does show up settles as -its number and counts as spent), and are listed on their own +its number, and the arm is then projected as a priced arm), and are listed on their own `PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms. The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start no new attempt, and release the attempts that never ran. An `unknown` settlement on an diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index baef02c92..21ae8b6e5 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -427,8 +427,9 @@ export function pilotReport(out: string, o: { total: number; workers?: number }) const record = JSON.parse(fs.readFileSync(path.join(out, PILOT_FILES.pilot), 'utf8')) as PilotRecord; const ledger = readLedger(path.join(out, PILOT_FILES.spend), record.cap_usd); const summary = summarize(ledger); - const seat = new Set(record.seat_metered ?? []); const armOf = new Map(record.attempts.map(a => [a.attempt, a.model])); + const charged = new Set(ledger.settled.filter(s => typeof s.usd === 'number' && s.usd > 0).map(s => armOf.get(s.attempt))); + const seat = new Set((record.seat_metered ?? []).filter(arm => !charged.has(arm))); const unknown = ledger.settled.filter(s => s.usd === 'unknown' && !seat.has(armOf.get(s.attempt) ?? '')).length; const workers = Math.max(1, o.workers ?? 1); const maxEstimate = Math.max(0, ...record.attempts.map(a => a.estimate_usd)); diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index ad302352d..5f78c45d0 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -219,7 +219,11 @@ describe('priced pilot', () => { const out = path.join(d, 'seat'); const s = startPilot({ out, cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry }); settlePilot(out, s.record.attempts[0]!.attempt, 0.05); - expect(pilotReport(out, { total: 1 }).spend.summary.spent_usd).toBeCloseTo(0.05, 6); + const charged = pilotReport(out, { total: 1000 }); + expect(charged.spend.summary.spent_usd).toBeCloseTo(0.05, 6); + expect(charged.seat_metered).toEqual([]); + expect(charged.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 1]]); + expect(charged.projection.verdict).toBe('unknown'); expect(() => startPilot({ out: path.join(d, 'stray'), cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-5.4-mini'], dryRun: dry })).toThrow('--seat-metered'); const switched = path.join(d, 'switched'); startPilot({ out: switched, cap: 1, n: 3, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });