diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 1dc081a76..f19260885 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -6,7 +6,7 @@ * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] - * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] + * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] [--seat-metered ] * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] @@ -50,9 +50,11 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json] stub model over the real rows; checklist: price table, output budget, retry prompt text, and with --cap reservation-fits-cap (one call: largest row + output budget, priced) - pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] + pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] [--seat-metered arm,...] + --seat-metered: arms on a subscription with no incremental charge; they reserve $0, + settle only unknown, and sit outside the cap and the projection pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id - pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask + pilot report --out dir --total items [--workers W] [--json] spend table + per-arm extrapolation with a 95% band, then the approval ask price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a @@ -168,7 +170,7 @@ function doPilot(rest: string[]): number { if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap --arms --dry-run '); const rules = lintModels(args, arms.map(model => ({ model, scope: scopeOf(args), where: 'pilot start --arms' })), 'pilot start'); if (!rules.ok) return EXIT.fail; - const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry }); + const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, seatMetered: csv(flag(args, '--seat-metered')), dryRun: dry }); if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; } for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`); process.stdout.write(`PILOT: status=${r.record.status} reserved=${r.record.attempts.length} cap=$${cap.toFixed(2)} ledger=${path.join(out, 'spend.json')}\n`); diff --git a/changelog.d/gst-566-eval-plan-seat-metered.md b/changelog.d/gst-566-eval-plan-seat-metered.md new file mode 100644 index 000000000..7cb227b7e --- /dev/null +++ b/changelog.d/gst-566-eval-plan-seat-metered.md @@ -0,0 +1,2 @@ +### Fixed +- **`/eval-plan` pilots with a subscription arm can now be approved, and the cost projection is per arm (#3653).** `gstack-eval-plan pilot start --seat-metered ` declares an arm that runs on a subscription with no extra charge (for example `claude -p` on a seat). Its attempts hold none of the spend cap, settle only as `unknown`, and get their own `PILOT_SEAT_METERED:` line, so the verdict covers the arms that cost money. `pilot report` now projects each arm from its own attempts (one `PILOT_ARM:` line each) and adds them up. Before, it averaged all arms together: a priced arm's cost was multiplied across arms with no price, and two arms at different prices got a band so wide that a pilot that fit the cap was told it exceeded it. The skill also says to stop a seat-metered run at its first quota or rate-limit error, with no retry (idea from #3273 by @merit-blake). diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 5ec40e511..5990d07d3 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -138,7 +138,7 @@ Write (or open) the preregistration file. Each section is required and must be f | Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | -| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | +| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | | No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | @@ -254,7 +254,15 @@ anything else (`EVAL_DRY_RUN_FAILED`). `--out`): admission is `spent + reserved + unknown + estimate <= cap`, under a file lock shared with any concurrent worker. Run each `PILOT_ATTEMPT:` through the project's own eval command, then settle it with the actual cost from the provider's usage field, or -`unknown` when the call was seat-metered or CLI-metered: +`unknown` when the call was seat-metered or CLI-metered. + +An arm that runs on a subscription with no incremental charge (for example `claude -p` on +a seat) is declared when the pilot starts: `--seat-metered `, each one also in +`--arms`. Its attempts reserve $0, settle only as `unknown`, and are listed on their own +`PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms. +The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start +no new attempt, and release the attempts that never ran. An `unknown` settlement on an +undeclared arm keeps `verdict=unknown`. Settle each attempt, then report: ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan pilot settle --out --attempt --usd @@ -263,8 +271,9 @@ eval command, then settle it with the actual cost from the provider's usage fiel `pilot report` prints the spend table (`spent`, `reserved`, `unknown charges`, the worst case with one in-flight call per worker, the worst-case overrun against the cap) and -`PILOT_PROJECTION:` — mean cost per item, its sample deviation, the projection to -`--total` items across the arms, and a 95% band. Regenerate the table at every later gate +one `PILOT_ARM:` line per money arm (its own mean, deviation, projection and band), then +`PILOT_PROJECTION:` — the arms summed: cost per item, deviation, the projection to +`--total` items, and a 95% band. Regenerate the table at every later gate of the run. Stop and report, without asking for approval, when the pilot missed a bar from the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 019499eca..e0dd2da88 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -60,7 +60,7 @@ Write (or open) the preregistration file. Each section is required and must be f | Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | -| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | +| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | | No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | @@ -176,7 +176,15 @@ anything else (`EVAL_DRY_RUN_FAILED`). `--out`): admission is `spent + reserved + unknown + estimate <= cap`, under a file lock shared with any concurrent worker. Run each `PILOT_ATTEMPT:` through the project's own eval command, then settle it with the actual cost from the provider's usage field, or -`unknown` when the call was seat-metered or CLI-metered: +`unknown` when the call was seat-metered or CLI-metered. + +An arm that runs on a subscription with no incremental charge (for example `claude -p` on +a seat) is declared when the pilot starts: `--seat-metered `, each one also in +`--arms`. Its attempts reserve $0, settle only as `unknown`, and are listed on their own +`PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms. +The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start +no new attempt, and release the attempts that never ran. An `unknown` settlement on an +undeclared arm keeps `verdict=unknown`. Settle each attempt, then report: ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan pilot settle --out --attempt --usd @@ -185,8 +193,9 @@ eval command, then settle it with the actual cost from the provider's usage fiel `pilot report` prints the spend table (`spent`, `reserved`, `unknown charges`, the worst case with one in-flight call per worker, the worst-case overrun against the cap) and -`PILOT_PROJECTION:` — mean cost per item, its sample deviation, the projection to -`--total` items across the arms, and a 95% band. Regenerate the table at every later gate +one `PILOT_ARM:` line per money arm (its own mean, deviation, projection and band), then +`PILOT_PROJECTION:` — the arms summed: cost per item, deviation, the projection to +`--total` items, and a 95% band. Regenerate the table at every later gate of the run. Stop and report, without asking for approval, when the pilot missed a bar from the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 6bb74db94..67ba6ebe0 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -366,15 +366,21 @@ function renderReservationCheck(c: NonNullable !o.arms.includes(a)); + if (stray.length) throw new Error(`--seat-metered ${stray.join(',')} needs to name arms listed in --arms`); + const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, ...(seat.length ? { seat_metered: seat } : {}), cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`; const keys = o.arms.map(keyOf); const duplicates = [...new Set(o.arms.filter((_, i) => keys.indexOf(keys[i]!) !== i).map(a => o.arms.filter(b => keyOf(b) === keyOf(a)).join(' = ')))]; @@ -383,11 +389,12 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: const ledgerFile = path.join(o.out, PILOT_FILES.spend); let error: { code: ResultCodeName; message: string } | undefined; outer: for (const model of o.arms) { - if (!hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; } + const seated = seat.includes(model); + if (!seated && !hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; } for (let i = 1; i <= o.n; i++) { - const estimate = estimateCostUsd(tokens, armModel(model)); + const estimate = seated ? 0 : estimateCostUsd(tokens, armModel(model)); const attempt = `pilot-${keyOf(model)}-${i}`; - try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}`, now: o.now }); } + try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}${seated ? ' (seat-metered)' : ''}`, now: o.now }); } catch (e) { if (e instanceof SpendCapExceeded) { record.status = 'budget_exhausted'; error = { code: 'EVAL_BUDGET_EXHAUSTED', message: e.message }; break outer; } throw e; @@ -400,6 +407,10 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: } export function settlePilot(out: string, attempt: string, usd: number | 'unknown'): SpendLedger { + const file = path.join(out, PILOT_FILES.pilot); + const record = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) as PilotRecord : null; + const arm = record?.attempts.find(a => a.attempt === attempt)?.model; + if (usd !== 'unknown' && arm !== undefined && record?.seat_metered?.includes(arm)) throw new Error(`${attempt} is on seat-metered arm ${arm}; a seat-metered attempt needs --usd unknown`); return settle(path.join(out, PILOT_FILES.spend), attempt, usd); } export function releasePilot(out: string, attempt: string): SpendLedger { @@ -408,31 +419,43 @@ export function releasePilot(out: string, attempt: string): SpendLedger { /** Two-sided 95% Student t for n-1 degrees of freedom (n settled items); 2.0 past the table. */ const T95: Record = { 2: 12.706, 3: 4.303, 4: 3.182, 5: 2.776, 6: 2.571, 7: 2.447, 8: 2.365, 9: 2.306, 10: 2.262 }; -export interface Projection { n: number; per_item_usd: number | null; sd_usd: number | null; total: number; projected_usd: number | null; band_usd: [number, number] | null; verdict: 'within' | 'exceeds' | 'unknown' } +export interface ArmProjection { arm: string; n: number; per_item_usd: number | null; sd_usd: number | null; projected_usd: number | null; band_usd: [number, number] | null } +export interface Projection { n: number; per_item_usd: number | null; sd_usd: number | null; total: number; projected_usd: number | null; band_usd: [number, number] | null; verdict: 'within' | 'exceeds' | 'unknown'; arms: ArmProjection[] } export interface SpendTable { summary: SpendSummary; workers: number; max_estimate_usd: number; worst_case_usd: number; worst_case_overrun_usd: number } -export interface PilotReport { record: PilotRecord; spend: SpendTable; projection: Projection; settled_known: number; settled_unknown: number } +export interface PilotReport { record: PilotRecord; spend: SpendTable; projection: Projection; settled_known: number; settled_unknown: number; seat_metered: Array<{ arm: string; calls: number; settled: number }> } export function pilotReport(out: string, o: { total: number; workers?: number }): PilotReport { const record = JSON.parse(fs.readFileSync(path.join(out, PILOT_FILES.pilot), 'utf8')) as PilotRecord; const ledger = readLedger(path.join(out, PILOT_FILES.spend), record.cap_usd); const summary = summarize(ledger); - const known = ledger.settled.filter(s => typeof s.usd === 'number').map(s => s.usd as number); - const unknown = ledger.settled.length - known.length; + const seat = new Set(record.seat_metered ?? []); + const armOf = new Map(record.attempts.map(a => [a.attempt, a.model])); + const unknown = ledger.settled.filter(s => s.usd === 'unknown' && !seat.has(armOf.get(s.attempt) ?? '')).length; const workers = Math.max(1, o.workers ?? 1); const maxEstimate = Math.max(0, ...record.attempts.map(a => a.estimate_usd)); const worst = summary.worst_case_usd + workers * maxEstimate; - const n = known.length; - const mean = n ? known.reduce((s, x) => s + x, 0) / n : null; - const sd = n > 1 && mean !== null ? Math.sqrt(known.reduce((s, x) => s + (x - mean) ** 2, 0) / (n - 1)) : null; - const perArmTotal = o.total * Math.max(1, record.arms.length); - const projected = mean === null ? null : mean * perArmTotal; - const half = sd !== null && mean !== null ? (T95[n] ?? 2.0) * (sd / Math.sqrt(n)) * perArmTotal : null; - const band: [number, number] | null = projected !== null && half !== null ? [Math.max(0, projected - half), projected + half] : null; - const verdict: Projection['verdict'] = projected === null || band === null || unknown > 0 ? 'unknown' : band[1] + summary.worst_case_usd <= record.cap_usd ? 'within' : 'exceeds'; + const arms: ArmProjection[] = record.arms.filter(arm => !seat.has(arm)).map(arm => { + const xs = ledger.settled.filter(s => armOf.get(s.attempt) === arm && typeof s.usd === 'number').map(s => s.usd as number); + const n = xs.length; + const mean = n ? xs.reduce((s, x) => s + x, 0) / n : null; + const sd = n > 1 && mean !== null ? Math.sqrt(xs.reduce((s, x) => s + (x - mean) ** 2, 0) / (n - 1)) : null; + const projected = mean === null ? null : mean * o.total; + const half = sd !== null ? (T95[n] ?? 2.0) * (sd / Math.sqrt(n)) * o.total : null; + return { arm, n, per_item_usd: mean, sd_usd: sd, projected_usd: projected, band_usd: projected !== null && half !== null ? [Math.max(0, projected - half), projected + half] : null }; + }); + const priced = arms.filter(a => a.per_item_usd !== null); + const n = priced.reduce((s, a) => s + a.n, 0); + const mean = priced.length || !arms.length ? priced.reduce((s, a) => s + a.per_item_usd!, 0) : null; + const sd = mean !== null && priced.every(a => a.sd_usd !== null) ? Math.sqrt(priced.reduce((s, a) => s + a.sd_usd! ** 2, 0)) : null; + const projected = mean === null ? null : mean * o.total; + const band: [number, number] | null = projected !== null && priced.every(a => a.band_usd) + ? [priced.reduce((s, a) => s + a.band_usd![0], 0), priced.reduce((s, a) => s + a.band_usd![1], 0)] : null; + const verdict: Projection['verdict'] = band === null || unknown > 0 || priced.length < arms.length ? 'unknown' : band[1] + summary.worst_case_usd <= record.cap_usd ? 'within' : 'exceeds'; return { record, settled_known: n, settled_unknown: unknown, spend: { summary, workers, max_estimate_usd: maxEstimate, worst_case_usd: worst, worst_case_overrun_usd: Math.max(0, worst - record.cap_usd) }, - projection: { n, per_item_usd: mean, sd_usd: sd, total: o.total, projected_usd: projected, band_usd: band, verdict }, + projection: { n, per_item_usd: mean, sd_usd: sd, total: o.total, projected_usd: projected, band_usd: band, verdict, arms }, + seat_metered: [...seat].map(arm => ({ arm, calls: record.attempts.filter(a => a.model === arm).length, settled: ledger.settled.filter(s => armOf.get(s.attempt) === arm).length })), }; } @@ -449,11 +472,17 @@ export function renderSpendTable(t: SpendTable, cap: number): string { export function renderPilotReport(r: PilotReport): string { const p = r.projection; + const band = (b: [number, number] | null) => (b ? `[${usd(b[0])}, ${usd(b[1])}]` : 'unknown'); + const notReady = r.settled_unknown > 0 + ? 'unknown charges make the band meaningless; settle them with actual cost, or restart the pilot with `pilot start --seat-metered ` for an arm on a subscription with no incremental charge' + : 'settle at least 2 attempts per priced arm with actual cost'; return [ `PILOT: n=${r.record.n} arms=${r.record.arms.join(',')} settled=${r.settled_known} unknown=${r.settled_unknown} status=${r.record.status}`, renderSpendTable(r.spend, r.record.cap_usd), - `PILOT_PROJECTION: items=${p.total} per_item=${usd(p.per_item_usd)} sd=${usd(p.sd_usd)} projected=${usd(p.projected_usd)} band=${p.band_usd ? `[${usd(p.band_usd[0])}, ${usd(p.band_usd[1])}]` : 'unknown'} cap=$${r.record.cap_usd.toFixed(2)} verdict=${p.verdict}`, - p.verdict === 'unknown' ? `APPROVAL: not ready — ${r.settled_known < 2 ? 'settle at least 2 pilot attempts with actual cost' : 'unknown charges make the band meaningless'}; settle, then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`, + ...p.arms.map(a => `PILOT_ARM: arm=${a.arm} n=${a.n} per_item=${usd(a.per_item_usd)} sd=${usd(a.sd_usd)} projected=${usd(a.projected_usd)} band=${band(a.band_usd)}`), + ...r.seat_metered.map(s => `PILOT_SEAT_METERED: arm=${s.arm} calls=${s.calls} settled=${s.settled} (declared seat-metered: no incremental charge, outside the cap and the projection; usage-limited)`), + `PILOT_PROJECTION: items=${p.total} known_arms=${p.arms.filter(a => a.per_item_usd !== null).length}/${p.arms.length} per_item=${usd(p.per_item_usd)} sd=${usd(p.sd_usd)} projected=${usd(p.projected_usd)} band=${band(p.band_usd)} cap=$${r.record.cap_usd.toFixed(2)} verdict=${p.verdict}`, + p.verdict === 'unknown' ? `APPROVAL: not ready — ${notReady}; then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`, ].join('\n') + '\n'; } diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index 8784d88a0..fab28a180 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -159,6 +159,68 @@ describe('priced pilot', () => { expect(r.settled_unknown).toBe(1); expect(r.projection.verdict).toBe('unknown'); }); + + test('#3653: each arm is projected on its own, so arms at different prices that fit the cap are not told they exceed it', () => { + const d = tmp(); + const out = path.join(d, 'mixed'); + const s = startPilot({ out, cap: 2, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) }); + expect(s.error).toBeUndefined(); + const usd: Record = { 'gpt-6-astra': [0.001, 0.001, 0.001], 'gpt-5.4-mini': [0.10, 0.11, 0.12] }; + for (const a of s.record.attempts) settlePilot(out, a.attempt, usd[a.model]![a.row - 1]!); + const r = pilotReport(out, { total: 10, workers: 1 }); + expect(r.projection.projected_usd).toBeCloseTo(1.11, 6); + expect(r.projection.band_usd![0]).toBeCloseTo(1.11 - 4.303 * 0.01 / Math.sqrt(3) * 10, 4); + expect(r.projection.band_usd![1]).toBeCloseTo(1.11 + 4.303 * 0.01 / Math.sqrt(3) * 10, 4); + expect(r.projection.verdict).toBe('within'); + expect(r.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 3], ['gpt-5.4-mini', 3]]); + expect(renderPilotReport(r)).toContain('PILOT_ARM: arm=gpt-5.4-mini n=3 per_item=$0.1100'); + }); + + test('#3653: a priced arm mean is never multiplied across an arm settled unknown', () => { + const d = tmp(); + const out = path.join(d, 'partial'); + const s = startPilot({ out, cap: 10, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) }); + for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 0.0003 : 'unknown'); + const r = pilotReport(out, { total: 400 }); + expect(r.projection.projected_usd).toBeCloseTo(0.12, 6); + expect(r.projection.verdict).toBe('unknown'); + expect(r.settled_unknown).toBe(3); + const text = renderPilotReport(r); + expect(text).toContain('known_arms=1/2'); + expect(text).toContain('--seat-metered'); + }); + + test('#3653: a declared seat-metered arm holds none of the cap, stays out of the verdict and gets its own line', () => { + const d = tmp(); + const dry = passingDryRun(d); + const est = startPilot({ out: path.join(d, 'probe'), cap: 100, n: 1, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: dry }).record.attempts; + const astra = est.find(a => a.model === 'gpt-6-astra')!.estimate_usd, mini = est.find(a => a.model === 'gpt-5.4-mini')!.estimate_usd; + const cap = Number((3 * mini + 2 * astra).toFixed(4)); + expect(startPilot({ out: path.join(d, 'undeclared'), cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], dryRun: dry }).error?.code).toBe('EVAL_BUDGET_EXHAUSTED'); + const out = path.join(d, 'seat'); + const s = startPilot({ out, cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry }); + expect(s.error).toBeUndefined(); + expect(s.record.seat_metered).toEqual(['gpt-6-astra']); + expect(s.record.attempts.filter(a => a.model === 'gpt-6-astra').map(a => a.estimate_usd)).toEqual([0, 0, 0]); + for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 'unknown' : 0.0003); + const r = pilotReport(out, { total: 400, workers: 3 }); + expect(r.settled_unknown).toBe(0); + expect(r.spend.summary.unknown_usd).toBe(0); + expect(r.spend.max_estimate_usd).toBeCloseTo(mini, 6); + expect(r.projection.projected_usd).toBeCloseTo(0.12, 6); + expect(r.projection.verdict).toBe('within'); + expect(r.seat_metered).toEqual([{ arm: 'gpt-6-astra', calls: 3, settled: 3 }]); + expect(renderPilotReport(r)).toContain('PILOT_SEAT_METERED: arm=gpt-6-astra calls=3 settled=3'); + }); + + test('#3653: a seat-metered attempt settles only unknown, and a seat-metered arm must be one of the arms', () => { + const d = tmp(); + const dry = passingDryRun(d); + const out = path.join(d, 'seat'); + const s = startPilot({ out, cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry }); + expect(() => settlePilot(out, s.record.attempts[0]!.attempt, 0.01)).toThrow('seat-metered'); + expect(() => startPilot({ out: path.join(d, 'stray'), cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-5.4-mini'], dryRun: dry })).toThrow('--seat-metered'); + }); }); // ---------------------------------------------------------------------------