fix(eval-plan): per-arm pilot projection and --seat-metered arms (#3653)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

bin/gstack-eval-plan

@@ -6,7 +6,7 @@
  *   gstack-eval-plan arms [--arms <model,model>] [--policy-doc <file>] [--input-tokens N] [--output-tokens N] [--json]
  *   gstack-eval-plan dry-run --input <rows.jsonl> --arms <model,model> [--fields a,b] [--output-budget N]
  *                            [--retry-prompt <file>] [--max-rows N] [--out <dir>] [--json]
- *   gstack-eval-plan pilot start  --out <dir> --cap <usd> --arms <model[#loadout],...> --dry-run <dry-run.json> [--n 3]
+ *   gstack-eval-plan pilot start  --out <dir> --cap <usd> --arms <model[#loadout],...> --dry-run <dry-run.json> [--n 3] [--seat-metered <arm,...>]
  *   gstack-eval-plan pilot settle --out <dir> --attempt <id> --usd <n|unknown>
  *   gstack-eval-plan pilot release --out <dir> --attempt <id>
  *   gstack-eval-plan pilot report --out <dir> --total <items> [--workers W] [--json]
@@ -50,9 +50,11 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
   dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json]
                                      stub model over the real rows; checklist: price table, output budget, retry prompt text,
                                      and with --cap reservation-fits-cap (one call: largest row + output budget, priced)
-  pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3]
+  pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] [--seat-metered arm,...]
+                                     --seat-metered: arms on a subscription with no incremental charge; they reserve $0,
+                                     settle only unknown, and sit outside the cap and the projection
   pilot settle --out dir --attempt id --usd <n|unknown>      pilot release --out dir --attempt id
-  pilot report --out dir --total items [--workers W] [--json]  spend table + extrapolation with a 95% band, then the approval ask
+  pilot report --out dir --total items [--workers W] [--json]  spend table + per-arm extrapolation with a 95% band, then the approval ask
   price-gate --gate-file <file> | --gate "<exact gate command>" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out dir] [--json]
                                      price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base
                                      policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
@@ -168,7 +170,7 @@ function doPilot(rest: string[]): number {
     if (cap === undefined || !arms.length || !dry || !fs.existsSync(dry)) return usage('pilot start needs --cap <usd> --arms <m,m> --dry-run <existing dry-run.json>');
     const rules = lintModels(args, arms.map(model => ({ model, scope: scopeOf(args), where: 'pilot start --arms' })), 'pilot start');
     if (!rules.ok) return EXIT.fail;
-    const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, dryRun: dry });
+    const r = startPilot({ out, cap, n: num(args, '--n', 3)!, arms, seatMetered: csv(flag(args, '--seat-metered')), dryRun: dry });
     if (r.error && !r.record) { process.stderr.write(resultLine('gstack-eval-plan pilot start', r.error.code, r.error.message) + '\n'); return EXIT.refused; }
     for (const a of r.record.attempts) process.stdout.write(`PILOT_ATTEMPT: ${a.attempt} model=${a.model} estimate=$${a.estimate_usd.toFixed(4)}\n`);
     process.stdout.write(`PILOT: status=${r.record.status} reserved=${r.record.attempts.length} cap=$${cap.toFixed(2)} ledger=${path.join(out, 'spend.json')}\n`);

changelog.d/gst-566-eval-plan-seat-metered.md

@@ -0,0 +1,2 @@
+### Fixed
+- **`/eval-plan` pilots with a subscription arm can now be approved, and the cost projection is per arm (#3653).** `gstack-eval-plan pilot start --seat-metered <arm>` declares an arm that runs on a subscription with no extra charge (for example `claude -p` on a seat). Its attempts hold none of the spend cap, settle only as `unknown`, and get their own `PILOT_SEAT_METERED:` line, so the verdict covers the arms that cost money. `pilot report` now projects each arm from its own attempts (one `PILOT_ARM:` line each) and adds them up. Before, it averaged all arms together: a priced arm's cost was multiplied across arms with no price, and two arms at different prices got a band so wide that a pilot that fit the cap was told it exceeded it. The skill also says to stop a seat-metered run at its first quota or rate-limit error, with no retry (idea from #3273 by @merit-blake).

eval-plan/SKILL.md

@@ -138,7 +138,7 @@ Write (or open) the preregistration file. Each section is required and must be f
 | Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b |
 | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
 | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
-| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` |
+| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm |
 | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
 | No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) |
 
@@ -254,7 +254,15 @@ anything else (`EVAL_DRY_RUN_FAILED`).
 `--out`): admission is `spent + reserved + unknown + estimate <= cap`, under a file lock
 shared with any concurrent worker. Run each `PILOT_ATTEMPT:` through the project's own
 eval command, then settle it with the actual cost from the provider's usage field, or
-`unknown` when the call was seat-metered or CLI-metered:
+`unknown` when the call was seat-metered or CLI-metered.
+
+An arm that runs on a subscription with no incremental charge (for example `claude -p` on
+a seat) is declared when the pilot starts: `--seat-metered <arm,...>`, each one also in
+`--arms`. Its attempts reserve $0, settle only as `unknown`, and are listed on their own
+`PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms.
+The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start
+no new attempt, and release the attempts that never ran. An `unknown` settlement on an
+undeclared arm keeps `verdict=unknown`. Settle each attempt, then report:
 
 ```bash
 ~/.claude/skills/gstack/bin/gstack-eval-plan pilot settle --out <outdir> --attempt <attempt-id> --usd <usd|unknown>
@@ -263,8 +271,9 @@ eval command, then settle it with the actual cost from the provider's usage fiel
 
 `pilot report` prints the spend table (`spent`, `reserved`, `unknown charges`, the worst case
 with one in-flight call per worker, the worst-case overrun against the cap) and
-`PILOT_PROJECTION:` — mean cost per item, its sample deviation, the projection to
-`--total` items across the arms, and a 95% band. Regenerate the table at every later gate
+one `PILOT_ARM:` line per money arm (its own mean, deviation, projection and band), then
+`PILOT_PROJECTION:` — the arms summed: cost per item, deviation, the projection to
+`--total` items, and a 95% band. Regenerate the table at every later gate
 of the run. Stop and report, without asking for approval, when the pilot missed a bar from
 the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
 

eval-plan/SKILL.md.tmpl

@@ -60,7 +60,7 @@ Write (or open) the preregistration file. Each section is required and must be f
 | Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b |
 | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
 | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
-| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` |
+| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm |
 | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
 | No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) |
 
@@ -176,7 +176,15 @@ anything else (`EVAL_DRY_RUN_FAILED`).
 `--out`): admission is `spent + reserved + unknown + estimate <= cap`, under a file lock
 shared with any concurrent worker. Run each `PILOT_ATTEMPT:` through the project's own
 eval command, then settle it with the actual cost from the provider's usage field, or
-`unknown` when the call was seat-metered or CLI-metered:
+`unknown` when the call was seat-metered or CLI-metered.
+
+An arm that runs on a subscription with no incremental charge (for example `claude -p` on
+a seat) is declared when the pilot starts: `--seat-metered <arm,...>`, each one also in
+`--arms`. Its attempts reserve $0, settle only as `unknown`, and are listed on their own
+`PILOT_SEAT_METERED:` line instead of the projection, so the verdict covers the money arms.
+The first quota or rate-limit error on a seat-metered arm ends the run: never retry it, start
+no new attempt, and release the attempts that never ran. An `unknown` settlement on an
+undeclared arm keeps `verdict=unknown`. Settle each attempt, then report:
 
 ```bash
 ~/.claude/skills/gstack/bin/gstack-eval-plan pilot settle --out <outdir> --attempt <attempt-id> --usd <usd|unknown>
@@ -185,8 +193,9 @@ eval command, then settle it with the actual cost from the provider's usage fiel
 
 `pilot report` prints the spend table (`spent`, `reserved`, `unknown charges`, the worst case
 with one in-flight call per worker, the worst-case overrun against the cap) and
-`PILOT_PROJECTION:` — mean cost per item, its sample deviation, the projection to
-`--total` items across the arms, and a 95% band. Regenerate the table at every later gate
+one `PILOT_ARM:` line per money arm (its own mean, deviation, projection and band), then
+`PILOT_PROJECTION:` — the arms summed: cost per item, deviation, the projection to
+`--total` items, and a 95% band. Regenerate the table at every later gate
 of the run. Stop and report, without asking for approval, when the pilot missed a bar from
 the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
 

lib/eval-plan.ts

@@ -366,15 +366,21 @@ function renderReservationCheck(c: NonNullable<DryRunResult['checks']['reservati
 // ---------------------------------------------------------------------------
 export const PILOT_FILES = { spend: 'spend.json', pilot: 'pilot.json' } as const;
 export interface PilotAttempt { attempt: string; model: string; row: number; estimate_usd: number }
-export interface PilotRecord { schema_version: 1; n: number; arms: string[]; cap_usd: number; dry_run: string; attempts: PilotAttempt[]; started_at: string; status: 'reserved' | 'budget_exhausted' }
-export interface PilotStartOptions { out: string; cap: number; n: number; arms: string[]; dryRun: string; tokens?: TokenEstimate; now?: Date }
+export interface PilotRecord { schema_version: 1; n: number; arms: string[]; seat_metered?: string[]; cap_usd: number; dry_run: string; attempts: PilotAttempt[]; started_at: string; status: 'reserved' | 'budget_exhausted' }
+export interface PilotStartOptions { out: string; cap: number; n: number; arms: string[]; seatMetered?: string[]; dryRun: string; tokens?: TokenEstimate; now?: Date }
 
-/** Reserve n attempts per arm under the cap; a refused reservation ends the pilot `budget_exhausted` with nothing else reserved. */
+/**
+ * Reserve n attempts per arm under the cap; a refused reservation ends the pilot `budget_exhausted` with nothing else reserved.
+ * A seat-metered arm (operator-declared: a subscription call with no incremental charge) reserves $0 and settles only `unknown`.
+ */
 export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: { code: ResultCodeName; message: string } } {
   const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult;
   if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } };
   const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget };
-  const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' };
+  const seat = o.seatMetered ?? [];
+  const stray = seat.filter(a => !o.arms.includes(a));
+  if (stray.length) throw new Error(`--seat-metered ${stray.join(',')} needs to name arms listed in --arms`);
+  const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, ...(seat.length ? { seat_metered: seat } : {}), cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' };
   const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`;
   const keys = o.arms.map(keyOf);
   const duplicates = [...new Set(o.arms.filter((_, i) => keys.indexOf(keys[i]!) !== i).map(a => o.arms.filter(b => keyOf(b) === keyOf(a)).join(' = ')))];
@@ -383,11 +389,12 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?:
   const ledgerFile = path.join(o.out, PILOT_FILES.spend);
   let error: { code: ResultCodeName; message: string } | undefined;
   outer: for (const model of o.arms) {
-    if (!hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; }
+    const seated = seat.includes(model);
+    if (!seated && !hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; }
     for (let i = 1; i <= o.n; i++) {
-      const estimate = estimateCostUsd(tokens, armModel(model));
+      const estimate = seated ? 0 : estimateCostUsd(tokens, armModel(model));
       const attempt = `pilot-${keyOf(model)}-${i}`;
-      try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}`, now: o.now }); }
+      try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}${seated ? ' (seat-metered)' : ''}`, now: o.now }); }
       catch (e) {
         if (e instanceof SpendCapExceeded) { record.status = 'budget_exhausted'; error = { code: 'EVAL_BUDGET_EXHAUSTED', message: e.message }; break outer; }
         throw e;
@@ -400,6 +407,10 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?:
 }
 
 export function settlePilot(out: string, attempt: string, usd: number | 'unknown'): SpendLedger {
+  const file = path.join(out, PILOT_FILES.pilot);
+  const record = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) as PilotRecord : null;
+  const arm = record?.attempts.find(a => a.attempt === attempt)?.model;
+  if (usd !== 'unknown' && arm !== undefined && record?.seat_metered?.includes(arm)) throw new Error(`${attempt} is on seat-metered arm ${arm}; a seat-metered attempt needs --usd unknown`);
   return settle(path.join(out, PILOT_FILES.spend), attempt, usd);
 }
 export function releasePilot(out: string, attempt: string): SpendLedger {
@@ -408,31 +419,43 @@ export function releasePilot(out: string, attempt: string): SpendLedger {
 
 /** Two-sided 95% Student t for n-1 degrees of freedom (n settled items); 2.0 past the table. */
 const T95: Record<number, number> = { 2: 12.706, 3: 4.303, 4: 3.182, 5: 2.776, 6: 2.571, 7: 2.447, 8: 2.365, 9: 2.306, 10: 2.262 };
-export interface Projection { n: number; per_item_usd: number | null; sd_usd: number | null; total: number; projected_usd: number | null; band_usd: [number, number] | null; verdict: 'within' | 'exceeds' | 'unknown' }
+export interface ArmProjection { arm: string; n: number; per_item_usd: number | null; sd_usd: number | null; projected_usd: number | null; band_usd: [number, number] | null }
+export interface Projection { n: number; per_item_usd: number | null; sd_usd: number | null; total: number; projected_usd: number | null; band_usd: [number, number] | null; verdict: 'within' | 'exceeds' | 'unknown'; arms: ArmProjection[] }
 export interface SpendTable { summary: SpendSummary; workers: number; max_estimate_usd: number; worst_case_usd: number; worst_case_overrun_usd: number }
-export interface PilotReport { record: PilotRecord; spend: SpendTable; projection: Projection; settled_known: number; settled_unknown: number }
+export interface PilotReport { record: PilotRecord; spend: SpendTable; projection: Projection; settled_known: number; settled_unknown: number; seat_metered: Array<{ arm: string; calls: number; settled: number }> }
 
 export function pilotReport(out: string, o: { total: number; workers?: number }): PilotReport {
   const record = JSON.parse(fs.readFileSync(path.join(out, PILOT_FILES.pilot), 'utf8')) as PilotRecord;
   const ledger = readLedger(path.join(out, PILOT_FILES.spend), record.cap_usd);
   const summary = summarize(ledger);
-  const known = ledger.settled.filter(s => typeof s.usd === 'number').map(s => s.usd as number);
-  const unknown = ledger.settled.length - known.length;
+  const seat = new Set(record.seat_metered ?? []);
+  const armOf = new Map(record.attempts.map(a => [a.attempt, a.model]));
+  const unknown = ledger.settled.filter(s => s.usd === 'unknown' && !seat.has(armOf.get(s.attempt) ?? '')).length;
   const workers = Math.max(1, o.workers ?? 1);
   const maxEstimate = Math.max(0, ...record.attempts.map(a => a.estimate_usd));
   const worst = summary.worst_case_usd + workers * maxEstimate;
-  const n = known.length;
-  const mean = n ? known.reduce((s, x) => s + x, 0) / n : null;
-  const sd = n > 1 && mean !== null ? Math.sqrt(known.reduce((s, x) => s + (x - mean) ** 2, 0) / (n - 1)) : null;
-  const perArmTotal = o.total * Math.max(1, record.arms.length);
-  const projected = mean === null ? null : mean * perArmTotal;
-  const half = sd !== null && mean !== null ? (T95[n] ?? 2.0) * (sd / Math.sqrt(n)) * perArmTotal : null;
-  const band: [number, number] | null = projected !== null && half !== null ? [Math.max(0, projected - half), projected + half] : null;
-  const verdict: Projection['verdict'] = projected === null || band === null || unknown > 0 ? 'unknown' : band[1] + summary.worst_case_usd <= record.cap_usd ? 'within' : 'exceeds';
+  const arms: ArmProjection[] = record.arms.filter(arm => !seat.has(arm)).map(arm => {
+    const xs = ledger.settled.filter(s => armOf.get(s.attempt) === arm && typeof s.usd === 'number').map(s => s.usd as number);
+    const n = xs.length;
+    const mean = n ? xs.reduce((s, x) => s + x, 0) / n : null;
+    const sd = n > 1 && mean !== null ? Math.sqrt(xs.reduce((s, x) => s + (x - mean) ** 2, 0) / (n - 1)) : null;
+    const projected = mean === null ? null : mean * o.total;
+    const half = sd !== null ? (T95[n] ?? 2.0) * (sd / Math.sqrt(n)) * o.total : null;
+    return { arm, n, per_item_usd: mean, sd_usd: sd, projected_usd: projected, band_usd: projected !== null && half !== null ? [Math.max(0, projected - half), projected + half] : null };
+  });
+  const priced = arms.filter(a => a.per_item_usd !== null);
+  const n = priced.reduce((s, a) => s + a.n, 0);
+  const mean = priced.length || !arms.length ? priced.reduce((s, a) => s + a.per_item_usd!, 0) : null;
+  const sd = mean !== null && priced.every(a => a.sd_usd !== null) ? Math.sqrt(priced.reduce((s, a) => s + a.sd_usd! ** 2, 0)) : null;
+  const projected = mean === null ? null : mean * o.total;
+  const band: [number, number] | null = projected !== null && priced.every(a => a.band_usd)
+    ? [priced.reduce((s, a) => s + a.band_usd![0], 0), priced.reduce((s, a) => s + a.band_usd![1], 0)] : null;
+  const verdict: Projection['verdict'] = band === null || unknown > 0 || priced.length < arms.length ? 'unknown' : band[1] + summary.worst_case_usd <= record.cap_usd ? 'within' : 'exceeds';
   return {
     record, settled_known: n, settled_unknown: unknown,
     spend: { summary, workers, max_estimate_usd: maxEstimate, worst_case_usd: worst, worst_case_overrun_usd: Math.max(0, worst - record.cap_usd) },
-    projection: { n, per_item_usd: mean, sd_usd: sd, total: o.total, projected_usd: projected, band_usd: band, verdict },
+    projection: { n, per_item_usd: mean, sd_usd: sd, total: o.total, projected_usd: projected, band_usd: band, verdict, arms },
+    seat_metered: [...seat].map(arm => ({ arm, calls: record.attempts.filter(a => a.model === arm).length, settled: ledger.settled.filter(s => armOf.get(s.attempt) === arm).length })),
   };
 }
 
@@ -449,11 +472,17 @@ export function renderSpendTable(t: SpendTable, cap: number): string {
 
 export function renderPilotReport(r: PilotReport): string {
   const p = r.projection;
+  const band = (b: [number, number] | null) => (b ? `[${usd(b[0])}, ${usd(b[1])}]` : 'unknown');
+  const notReady = r.settled_unknown > 0
+    ? 'unknown charges make the band meaningless; settle them with actual cost, or restart the pilot with `pilot start --seat-metered <arm>` for an arm on a subscription with no incremental charge'
+    : 'settle at least 2 attempts per priced arm with actual cost';
   return [
     `PILOT: n=${r.record.n} arms=${r.record.arms.join(',')} settled=${r.settled_known} unknown=${r.settled_unknown} status=${r.record.status}`,
     renderSpendTable(r.spend, r.record.cap_usd),
-    `PILOT_PROJECTION: items=${p.total} per_item=${usd(p.per_item_usd)} sd=${usd(p.sd_usd)} projected=${usd(p.projected_usd)} band=${p.band_usd ? `[${usd(p.band_usd[0])}, ${usd(p.band_usd[1])}]` : 'unknown'} cap=$${r.record.cap_usd.toFixed(2)} verdict=${p.verdict}`,
-    p.verdict === 'unknown' ? `APPROVAL: not ready — ${r.settled_known < 2 ? 'settle at least 2 pilot attempts with actual cost' : 'unknown charges make the band meaningless'}; settle, then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`,
+    ...p.arms.map(a => `PILOT_ARM: arm=${a.arm} n=${a.n} per_item=${usd(a.per_item_usd)} sd=${usd(a.sd_usd)} projected=${usd(a.projected_usd)} band=${band(a.band_usd)}`),
+    ...r.seat_metered.map(s => `PILOT_SEAT_METERED: arm=${s.arm} calls=${s.calls} settled=${s.settled} (declared seat-metered: no incremental charge, outside the cap and the projection; usage-limited)`),
+    `PILOT_PROJECTION: items=${p.total} known_arms=${p.arms.filter(a => a.per_item_usd !== null).length}/${p.arms.length} per_item=${usd(p.per_item_usd)} sd=${usd(p.sd_usd)} projected=${usd(p.projected_usd)} band=${band(p.band_usd)} cap=$${r.record.cap_usd.toFixed(2)} verdict=${p.verdict}`,
+    p.verdict === 'unknown' ? `APPROVAL: not ready — ${notReady}; then rerun report` : `APPROVAL: run ${p.total} items at projected ${usd(p.projected_usd)} (95% band ${usd(p.band_usd![0])}–${usd(p.band_usd![1])}) under cap $${r.record.cap_usd.toFixed(2)}: ${p.verdict === 'within' ? 'fits; ask the owner before any further priced call' : 'EXCEEDS the cap; stop, raise the cap on purpose or cut arms or items'}`,
   ].join('\n') + '\n';
 }
 

test/pricing-eval-plan.test.ts

@@ -159,6 +159,68 @@ describe('priced pilot', () => {
     expect(r.settled_unknown).toBe(1);
     expect(r.projection.verdict).toBe('unknown');
   });
+
+  test('#3653: each arm is projected on its own, so arms at different prices that fit the cap are not told they exceed it', () => {
+    const d = tmp();
+    const out = path.join(d, 'mixed');
+    const s = startPilot({ out, cap: 2, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
+    expect(s.error).toBeUndefined();
+    const usd: Record<string, number[]> = { 'gpt-6-astra': [0.001, 0.001, 0.001], 'gpt-5.4-mini': [0.10, 0.11, 0.12] };
+    for (const a of s.record.attempts) settlePilot(out, a.attempt, usd[a.model]![a.row - 1]!);
+    const r = pilotReport(out, { total: 10, workers: 1 });
+    expect(r.projection.projected_usd).toBeCloseTo(1.11, 6);
+    expect(r.projection.band_usd![0]).toBeCloseTo(1.11 - 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
+    expect(r.projection.band_usd![1]).toBeCloseTo(1.11 + 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
+    expect(r.projection.verdict).toBe('within');
+    expect(r.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 3], ['gpt-5.4-mini', 3]]);
+    expect(renderPilotReport(r)).toContain('PILOT_ARM: arm=gpt-5.4-mini n=3 per_item=$0.1100');
+  });
+
+  test('#3653: a priced arm mean is never multiplied across an arm settled unknown', () => {
+    const d = tmp();
+    const out = path.join(d, 'partial');
+    const s = startPilot({ out, cap: 10, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
+    for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 0.0003 : 'unknown');
+    const r = pilotReport(out, { total: 400 });
+    expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
+    expect(r.projection.verdict).toBe('unknown');
+    expect(r.settled_unknown).toBe(3);
+    const text = renderPilotReport(r);
+    expect(text).toContain('known_arms=1/2');
+    expect(text).toContain('--seat-metered');
+  });
+
+  test('#3653: a declared seat-metered arm holds none of the cap, stays out of the verdict and gets its own line', () => {
+    const d = tmp();
+    const dry = passingDryRun(d);
+    const est = startPilot({ out: path.join(d, 'probe'), cap: 100, n: 1, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: dry }).record.attempts;
+    const astra = est.find(a => a.model === 'gpt-6-astra')!.estimate_usd, mini = est.find(a => a.model === 'gpt-5.4-mini')!.estimate_usd;
+    const cap = Number((3 * mini + 2 * astra).toFixed(4));
+    expect(startPilot({ out: path.join(d, 'undeclared'), cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], dryRun: dry }).error?.code).toBe('EVAL_BUDGET_EXHAUSTED');
+    const out = path.join(d, 'seat');
+    const s = startPilot({ out, cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
+    expect(s.error).toBeUndefined();
+    expect(s.record.seat_metered).toEqual(['gpt-6-astra']);
+    expect(s.record.attempts.filter(a => a.model === 'gpt-6-astra').map(a => a.estimate_usd)).toEqual([0, 0, 0]);
+    for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 'unknown' : 0.0003);
+    const r = pilotReport(out, { total: 400, workers: 3 });
+    expect(r.settled_unknown).toBe(0);
+    expect(r.spend.summary.unknown_usd).toBe(0);
+    expect(r.spend.max_estimate_usd).toBeCloseTo(mini, 6);
+    expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
+    expect(r.projection.verdict).toBe('within');
+    expect(r.seat_metered).toEqual([{ arm: 'gpt-6-astra', calls: 3, settled: 3 }]);
+    expect(renderPilotReport(r)).toContain('PILOT_SEAT_METERED: arm=gpt-6-astra calls=3 settled=3');
+  });
+
+  test('#3653: a seat-metered attempt settles only unknown, and a seat-metered arm must be one of the arms', () => {
+    const d = tmp();
+    const dry = passingDryRun(d);
+    const out = path.join(d, 'seat');
+    const s = startPilot({ out, cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
+    expect(() => settlePilot(out, s.record.attempts[0]!.attempt, 0.01)).toThrow('seat-metered');
+    expect(() => startPilot({ out: path.join(d, 'stray'), cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-5.4-mini'], dryRun: dry })).toThrow('--seat-metered');
+  });
 });
 
 // ---------------------------------------------------------------------------