diff --git a/docs/ship-budget.md b/docs/ship-budget.md index e61d0b59e..32e64264f 100644 --- a/docs/ship-budget.md +++ b/docs/ship-budget.md @@ -227,6 +227,12 @@ From then on the run projects steps and dispatches with its own factor: the larg actual/prior ratio among its last three measurements of that kind, times 1.5. The step factor also scales the default tail reserve. Dispatches keep the prior until the first dispatch is measured, so under a tight cap admit one reviewer first and let it calibrate the rest. +On the Capy meter a step's prior is its calibrated call count (`step_calls`) times the run's own +median per-call charge over its last five measured calls (each `--reading` minus a fresh +`--prior-reading`), once three are measured, unscaled (`calibration=…,call:`); until then +it uses the calibrated `per_call`. A step prior never carries a dispatch: each audit or reviewer +batch is reserved by its own check, and a child's segment estimate counts them through +`segment_dispatches`. `close` prints `BUDGET_DRIFT:` with the run's projected and actual totals per kind. The estimate meter has no readings, so it always projects at the prior. diff --git a/lib/ship-budget-calibration.json b/lib/ship-budget-calibration.json index cd4d1517a..17f68ee0f 100644 --- a/lib/ship-budget-calibration.json +++ b/lib/ship-budget-calibration.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "note": "GSTA-152 plan item 8. Per-meter constants for /ship budget projections, fitted conservatively from metered runs: each constant is the largest observation times `headroom`, rounded up to `round_to` (lib/ship-budget.ts fitConstant). test/ship-budget-calibration.test.ts checks the constants equal the fit and that a leave-one-out fit on two runs is at or above the meter on the third. Savings claims count completed ships only. Each unit's call_floor (GSTA-167) is the host's per-call floor: what one call costs in a fresh session with the ship skill loaded, before any /ship work. The delegate thresholds compare the per-call charge minus this floor, so a host's own system prompt and tools never trigger delegation. A floor is the largest round_to multiple at or below its smallest observed fresh call, so it never hides real context growth; start --call-floor overrides it for one run.", + "note": "GSTA-152 plan item 8. Per-meter constants for /ship budget projections, fitted conservatively from metered runs: each constant is the largest observation times `headroom`, rounded up to `round_to` (lib/ship-budget.ts fitConstant). test/ship-budget-calibration.test.ts checks the constants equal the fit and that a leave-one-out fit on two runs is at or above the meter on the third. Savings claims count completed ships only. Each unit's call_floor (GSTA-167) is the host's per-call floor: what one call costs in a fresh session with the ship skill loaded, before any /ship work. The delegate thresholds compare the per-call charge minus this floor, so a host's own system prompt and tools never trigger delegation. A floor is the largest round_to multiple at or below its smallest observed fresh call, so it never hides real context growth; start --call-floor overrides it for one run. Step priors (GSTA-167 round 3): a step holds only its own calls; each audit dispatch (Steps 7, 8, 14.5) is reserved by its own check, so it moved from step_dispatches to segment_dispatches, where a child's segment estimate still counts it. On the Capy meter a step is priced at step_calls x the run's own median measured per-call charge once three calls are measured (lib/ship-budget.ts measuredPerCall), and at per_call before that. A keyed step_calls entry is the ceiling of the most calls step_calls_observed holds for it, and an unkeyed step's observations stay at or below default. Each observation is a step window's measured actual (step samples in the round-3 replay ledgers mirrored under the project-user Drive folder ship-budget/) divided by 330k, the per-call charge replay3-34.md reports for gstack-evals #34; replay3-36.md reports no per-call figure, so #36 uses the same 330k. Steps 7 and 9 have no step measurement yet and keep 12. The #3633 lane's first ledger (ship-20261030T020246-7c4b65) read the root counter, which holds sibling threads, so its step samples are left out.", "headroom": 1.5, "round_to": 10000, "observations": { @@ -19,6 +19,16 @@ "run": "gstack-evals#31", "value": 500000, "source": "GSTA-152 plan v3 evidence: the /ship child's context was about 0.5M per call at the handoff" + }, + { + "run": "gstack-evals#34 round 3", + "value": 330000, + "source": "handoffs/GSTA-167/replay3-34.md: about 0.33M per late parent call, read from the replay thread's own counter" + }, + { + "run": "GSTA-167 lane #3633", + "value": 350000, + "source": "GSTA-167 lane report, own-jam ledger ship-20261030T022114-ef9f9a: the lane's own calls run about 0.35M each (stated by the lane, not solved from the ledger)" } ], "per_dispatch": [ @@ -48,18 +58,60 @@ }, "step_calls": { "7": 12, - "8": 12, + "8": 13, "9": 12, + "14": 5, "default": 3 }, - "step_dispatches": { + "step_calls_observed": { + "8": [ + { + "run": "gstack-evals#34 round 3", + "value": 12.16, + "source": "ledger ship-20261030T020705-eda850, step sample step 7 next 8 (the child's first step window, section reads included): actual 4012570 / 330k per call" + }, + { + "run": "gstack-evals#36 round 3", + "value": 1.02, + "source": "ledger ship-20261030T020847-c940ef, step sample step 7 next 8: actual 335631 / 330k per call" + } + ], + "13": [ + { + "run": "gstack-evals#34 round 3", + "value": 1.31, + "source": "ledger ship-20261030T021912-ab78e5, step sample step 12 next 13: actual 431321 / 330k per call" + } + ], + "14": [ + { + "run": "gstack-evals#34 round 3", + "value": 4.24, + "source": "ledger ship-20261030T021912-ab78e5, step sample step 13 next 14: actual 1399070 / 330k per call" + } + ], + "15": [ + { + "run": "gstack-evals#34 round 3", + "value": 2.4, + "source": "ledger ship-20261030T021912-ab78e5, step sample step 14.5 next 15: actual 790578 / 330k per call" + } + ], + "14.5": [ + { + "run": "gstack-evals#34 round 3", + "value": 1.55, + "source": "ledger ship-20261030T021912-ab78e5, step sample step 14 next 14.5: actual 510368 / 330k per call" + } + ] + }, + "step_dispatches": {}, + "segment_dispatches": { "7": 1, "8": 1, - "14.5": 1 - }, - "segment_dispatches": { "9": 4, - "11": 2 + "11": 2, + "14.5": 1 }, "units": { "capy-tokens": { diff --git a/lib/ship-budget-cli.ts b/lib/ship-budget-cli.ts index b87030144..e08ce9890 100644 --- a/lib/ship-budget-cli.ts +++ b/lib/ship-budget-cli.ts @@ -75,6 +75,7 @@ export function readMeter(l: Ledger, o: Opts, requireReading: boolean, notes: st notes.push(`CALL_UNKNOWN: --prior-reading ${prior} is older than the run's last reading ${last}; it spans several calls, so the per-call charge is unknown (pass the reading your previous call returned)`); return null; } + (l.calls ??= []).push({ epoch: l.meter.epoch, value: reading - prior }); return reading - prior; } diff --git a/lib/ship-budget.ts b/lib/ship-budget.ts index 6c7946c13..9b8cae7f6 100644 --- a/lib/ship-budget.ts +++ b/lib/ship-budget.ts @@ -79,6 +79,8 @@ export interface Ledger { cap_raises: Array<{ from: number | null; to: number | null; by: string; words: string; at: string }>; marker?: string; review_log?: string; calibration?: string; samples?: Sample[]; tail_explicit?: boolean; call_floor?: number; depth?: number; steps_done?: string[]; + /** Capy per-call charges (--reading minus a fresh --prior-reading), per epoch: step priors use their median. */ + calls?: Array<{ epoch: number; value: number }>; stopped?: { at: string; reason: string; step: string; next: string }; closed_at?: string; } @@ -123,6 +125,7 @@ export interface Calibration { observations: Record>; constants: Record<'per_call' | 'per_dispatch' | 'stop_overhead', number>; step_calls: Record; step_dispatches: Record; segment_dispatches: Record; + step_calls_observed?: Record>; units: Record; } export const CALIBRATION_FILE = path.join(import.meta.dir, 'ship-budget-calibration.json'); @@ -144,9 +147,24 @@ export function unitConstants(cal: Calibration, unit: string): Calibration['cons return entry.seeded_from ? unitConstants(cal, entry.seeded_from) : cal.constants; } -export function stepProjection(cal: Calibration, unit: string, step: string): number { - const k = unitConstants(cal, unit); - return (cal.step_calls[step] ?? cal.step_calls.default ?? 0) * k.per_call + (cal.step_dispatches[step] ?? 0) * k.per_dispatch; +export const stepCalls = (cal: Calibration, step: string) => cal.step_calls[step] ?? cal.step_calls.default ?? 0; + +/** A step's prior: its calibrated calls at `perCall` (the unit's per_call by default) plus any dispatches the step itself carries. */ +export function stepProjection(cal: Calibration, unit: string, step: string, perCall = unitConstants(cal, unit).per_call): number { + return stepCalls(cal, step) * perCall + (cal.step_dispatches[step] ?? 0) * unitConstants(cal, unit).per_dispatch; +} + +/** Calls the per-call median looks at, and how many it needs before it replaces the calibrated per_call. */ +export const CALL_WINDOW = 5; +export const MIN_CALLS = 3; + +/** The Capy run's median per-call charge over its last CALL_WINDOW measured calls of this epoch; null below MIN_CALLS or on another meter. */ +export function measuredPerCall(l: Ledger): number | null { + if (l.meter.contract.meter !== 'capy') return null; + const xs = (l.calls ?? []).filter(c => c.epoch === l.meter.epoch).slice(-CALL_WINDOW).map(c => c.value).sort((a, b) => a - b); + if (xs.length < MIN_CALLS) return null; + const mid = Math.floor(xs.length / 2); + return xs.length % 2 ? xs[mid]! : Math.round((xs[mid - 1]! + xs[mid]!) / 2); } /** Calls a packet reader makes: the parallel range reads, at most one turn of extra ranges, one write. */ @@ -180,6 +198,9 @@ export function measuredFactor(l: Ledger, cal: Calibration, kind: Sample['kind'] * with the step factor unless --reserve set it; the stop overhead keeps its * calibrated fit so a run always has room to stop. `floor` is the host's * per-call floor: start --call-floor, else the unit's calibrated value. + * Once a Capy run has MIN_CALLS measured calls, a step is priced at its + * calibrated calls x their median (measuredPerCall) instead, unscaled: that + * median is already the run's own measurement. * `reader` prices one dispatch reader: its prior (packet-based when the bytes are * known) times the dispatch factor, but never above the larger of that prior and * the run's clean per-reader mean, and never below the role's own measured mean. @@ -201,15 +222,20 @@ export function projector(l: Ledger, cal: Calibration = loadCalibration(l.calibr const scaled = scale(prior, dispatch); return Math.max(clean ? Math.min(scaled, Math.max(prior, clean)) : scaled, mean(measured.get(role) ?? [])); }; + const perCall = measuredPerCall(l); const project = (next: Next) => { if (next.kind === 'dispatch') return { prior: projection(cal, unit, next, floor), tokens: next.roles.reduce((n, role, i) => n + reader(role, next.bytes?.[i]), 0) }; + if (perCall !== null) { + const prior = stepProjection(cal, unit, next.step, perCall); + return { prior, tokens: prior }; + } const prior = projection(cal, unit, next); return { prior, tokens: scale(prior, step) }; }; const segment = (from: string, to: string) => SHIP_STEPS.slice(SHIP_STEPS.indexOf(from as any), SHIP_STEPS.indexOf(to as any) + 1) .reduce((sum, s) => sum + project({ kind: 'step', step: s }).tokens + (cal.segment_dispatches[s] ?? 0) * project({ kind: 'dispatch', roles: ['specialist'] }).tokens, 0); const label = (f: number | null) => (f === null ? 'prior' : f.toFixed(3)); - return { project, segment, reader, reserve: (l.tail_explicit ? l.tail : scale(l.tail, step)) + l.stop_overhead, label: `step:${label(step)},dispatch:${label(dispatch)}`, floor }; + return { project, segment, reader, reserve: (l.tail_explicit ? l.tail : scale(l.tail, step)) + l.stop_overhead, label: `step:${label(step)},dispatch:${label(dispatch)}${perCall === null ? '' : `,call:${perCall}`}`, floor }; } /** @@ -289,7 +315,7 @@ export const SMALL_RUN_CALL = 250_000; export function smallRunSegment(cal: Calibration, from: string, to: string, call: number | null): number { const steps = SHIP_STEPS.slice(SHIP_STEPS.indexOf(from as any), SHIP_STEPS.indexOf(to as any) + 1); - return steps.reduce((n, s) => n + (cal.step_calls[s] ?? cal.step_calls.default ?? 0), 0) * (call ?? SMALL_RUN_CALL); + return steps.reduce((n, s) => n + stepCalls(cal, s), 0) * (call ?? SMALL_RUN_CALL); } /** The SMALL_RUN line when a segment boundary should continue instead of delegating; null keeps the delegate rule. */ diff --git a/ship/sections/budget-handoff.md b/ship/sections/budget-handoff.md index de425fa27..8133ef88f 100644 --- a/ship/sections/budget-handoff.md +++ b/ship/sections/budget-handoff.md @@ -12,7 +12,7 @@ Field-by-field reference: docs/ship-budget.md. Exit 0 continue or delegate, 3 st - Capy: read `(await tools.thread.get({ thread_id: "" })).cost.tokens` (your own jam, its subagents included; a root thread's is `"this"`) in the same program as the `gstack-ship-budget` command, never its own call: `--reading `. `--prior-reading ` (per-call - charge) is your previous call's reading; one older than the run's last reading is ignored. + charge; three of them price steps at the run's own median) is your previous call's reading; one older than the run's last reading is ignored. - Claude Code: `start --session `, or echo the nonce the refusal prints once and rerun `start --nonce `; the helper reads the journals itself (never `--reading`). - Elsewhere: `--meter estimate` (printed `meter=estimate`), only when the owner accepts it. diff --git a/ship/sections/budget-handoff.md.tmpl b/ship/sections/budget-handoff.md.tmpl index d9b51c198..48887add4 100644 --- a/ship/sections/budget-handoff.md.tmpl +++ b/ship/sections/budget-handoff.md.tmpl @@ -10,7 +10,7 @@ Field-by-field reference: docs/ship-budget.md. Exit 0 continue or delegate, 3 st - Capy: read `(await tools.thread.get({ thread_id: "" })).cost.tokens` (your own jam, its subagents included; a root thread's is `"this"`) in the same program as the `gstack-ship-budget` command, never its own call: `--reading `. `--prior-reading ` (per-call - charge) is your previous call's reading; one older than the run's last reading is ignored. + charge; three of them price steps at the run's own median) is your previous call's reading; one older than the run's last reading is ignored. - Claude Code: `start --session `, or echo the nonce the refusal prints once and rerun `start --nonce `; the helper reads the journals itself (never `--reading`). - Elsewhere: `--meter estimate` (printed `meter=estimate`), only when the owner accepts it. diff --git a/test/ship-budget-calibration.test.ts b/test/ship-budget-calibration.test.ts index 2d09e0d57..a52f274b3 100644 --- a/test/ship-budget-calibration.test.ts +++ b/test/ship-budget-calibration.test.ts @@ -29,6 +29,19 @@ describe('ship-budget calibration', () => { } }); + // Value: protects=step_calls are refit from measured step windows, not guessed: a keyed step equals the ceiling of its most observed calls and an unkeyed step stays at or under default; fails_when=a step_calls entry drifts from its observations or loses its sources; + // why_new=the constant tests above never covered step_calls; seam=none; no_claim=the 330k per-call divisor fits every run + test('step_calls equal the ceiling of their most observed calls; unkeyed steps stay at or under default', () => { + const observed = cal.step_calls_observed ?? {}; + expect(Object.keys(observed)).toEqual(expect.arrayContaining(['8', '14'])); + for (const [step, rows] of Object.entries(observed)) { + for (const row of rows) expect(row.source.length, step).toBeGreaterThan(20); + const most = Math.ceil(Math.max(...rows.map(r => r.value))); + if (step in cal.step_calls) expect(cal.step_calls[step], step).toBe(most); + else expect(most, step).toBeLessThanOrEqual(cal.step_calls.default!); + } + }); + test('every meter unit has its own per-call floor: the largest round_to multiple at or below its observed fresh call, justified', () => { for (const unit of ['capy-tokens', 'claude-usage-tokens', 'estimate-tokens']) { const u = cal.units[unit]!; diff --git a/test/ship-budget-projection.test.ts b/test/ship-budget-projection.test.ts index b223b0e64..cca2b9d37 100644 --- a/test/ship-budget-projection.test.ts +++ b/test/ship-budget-projection.test.ts @@ -39,7 +39,7 @@ const LEAN_DISPATCH = 500_000; const STOP_26 = 1_650_000; describe('projection calibrated from the run\'s own meter (GST-101)', () => { - test('GSTA-2 replay: an 8M cap admits Steps 6 and 6.5 after Steps 1-5 cost 0.27M each, and stops with room to stop', () => { + test('GSTA-2 replay: an 8M cap admits Steps 6, 6.5 and 7 after Steps 1-5 cost 0.27M each, and stops with room to stop', () => { const h = fresh(); const cap = 8_000_000; const s = started(h, '8M'); @@ -49,7 +49,7 @@ describe('projection calibrated from the run\'s own meter (GST-101)', () => { let stop = ''; for (let i = 0; i + 1 < steps.length; i++) { const calls = cal.step_calls[steps[i]!] ?? cal.step_calls.default!; - reading += Math.round((GSTA2_STEP / 3) * calls * (1 + 0.05 * i)) + (cal.step_dispatches[steps[i]!] ?? 0) * LEAN_DISPATCH; + reading += Math.round((GSTA2_STEP / 3) * calls * (1 + 0.05 * i)) + (['7', '8', '14.5'].includes(steps[i]!) ? (cal.segment_dispatches[steps[i]!] ?? 0) * LEAN_DISPATCH : 0); expect(reading, `spend after Step ${steps[i]}`).toBeLessThanOrEqual(cap); const c = h.sb(['check', '--run', s.run, '--step', steps[i]!, '--next', steps[i + 1]!, '--reading', String(reading)]); if (c.code === 3) { stop = c.out; break; } @@ -57,7 +57,8 @@ describe('projection calibrated from the run\'s own meter (GST-101)', () => { admitted.push(steps[i + 1]!); if (steps[i + 1] === '6') expect(Number(h.field(c.out, 'BUDGET', 'projected'))).toBeLessThan(1_000_000); } - expect(admitted).toEqual(['2', '3', '4', '5', '6', '6.5']); + // GSTA-167 round 3: Step 7's prior no longer double-counts its audit dispatch (reserved by its own check), so Step 7 fits too. + expect(admitted).toEqual(['2', '3', '4', '5', '6', '6.5', '7']); expect(stop).toContain('STOP: reason=budget_exhausted'); expect(reading + GSTA2_STOP).toBeLessThanOrEqual(cap); }); diff --git a/test/ship-budget-step-prior.test.ts b/test/ship-budget-step-prior.test.ts new file mode 100644 index 000000000..c9d553908 --- /dev/null +++ b/test/ship-budget-step-prior.test.ts @@ -0,0 +1,50 @@ +/** + * GSTA-167 round 3: step priors were far too high. Step 8 was projected at + * 16.43M (12 calls x the calibrated 750k per call + one 7.43M dispatch) and + * cost 0.34M on gstack-evals #36 (ledger ship-20261030T020847-c940ef), and that + * inflation is what blocks capped runs. On the Capy meter (the agent's own jam + * counter) a step is priced at its calibrated call count times the run's own + * median measured per-call charge (--prior-reading), once three calls are + * measured; its dispatches are reserved by their own check, so a step prior + * carries no dispatch. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { loadCalibration, stepProjection } from '../lib/ship-budget'; +import { budgetHarness, type Harness } from './helpers/ship-budget-harness'; + +const cal = loadCalibration(); +const stepCalls = (c: typeof cal, step: string) => c.step_calls[step] ?? c.step_calls.default ?? 0; +const harnesses: Harness[] = []; +const fresh = () => { const h = budgetHarness(); harnesses.push(h); return h; }; +afterEach(() => { while (harnesses.length) harnesses.pop()!.cleanup(); }); + +describe('step priors from the run\'s own per-call charge', () => { + // Value: protects=after three measured calls a step is priced at its calibrated calls x the run's median per-call charge, so #36's Step 8 no longer reserves 16.43M; fails_when=the step prior keeps the calibrated per_call (or a dispatch) after the run has measured its calls; + // why_new=no test fed per-call charges into a step projection; seam=none; no_claim=that the calibrated call counts fit every diff + test('#36 replay: measured calls of 300k, 340k, 360k and 340k price Step 8 at its calls x their 340k median', () => { + const h = fresh(); + const s = h.sb(['start', '--record', h.record(), '--token-cap', '40M', '--reading', '1300000', '--prior-reading', '1000000']); + expect(s.code, s.out + s.err).toBe(0); + const run = h.field(s.out, 'BUDGET_RUN', 'id'); + expect(h.field(s.out, 'BUDGET', 'projected')).toBe(String(stepProjection(cal, 'capy-tokens', '1'))); + const a = h.sb(['check', '--run', run, '--step', '3', '--next', '4', '--reading', '1640000', '--prior-reading', '1300000']); + expect(h.field(a.out, 'BUDGET', 'call')).toBe('340000'); + const b = h.sb(['check', '--run', run, '--step', '6', '--next', '7', '--reading', '2000000', '--prior-reading', '1640000']); + expect(b.code, b.out + b.err).toBe(0); + const c = h.sb(['check', '--run', run, '--step', '7', '--next', '8', '--reading', '2440000', '--prior-reading', '2100000']); + expect(c.code, c.out + c.err).toBe(0); + expect(h.field(c.out, 'BUDGET', 'projected')).toBe(String(stepCalls(cal, '8') * 340_000)); + expect(h.field(c.out, 'BUDGET', 'calibration')).toEndWith(',call:340000'); + expect(Number(h.field(c.out, 'BUDGET', 'projected'))).toBeLessThan(5_000_000); + }); + + // Value: protects=a step prior holds only the step's own calls: the audits of Steps 7, 8 and 14.5 are reserved by their own dispatch check, and a child's segment estimate still counts them; fails_when=step_dispatches prices a dispatch into the step (double counted with its reservation) or the segment estimate drops it; + // why_new=no test separated the step prior from its dispatches; seam=none; no_claim=the audit always runs + test('a step prior carries no dispatch; the segment estimate keeps the audits', () => { + const k = cal.constants; + for (const step of ['7', '8', '14.5']) { + expect(stepProjection(cal, 'capy-tokens', step), step).toBe(stepCalls(cal, step) * k.per_call); + expect(cal.segment_dispatches[step] ?? 0, step).toBeGreaterThanOrEqual(1); + } + }); +}); diff --git a/test/size-ledger/ship/gsta-167-pr34.json b/test/size-ledger/ship/gsta-167-pr34.json index 5c66f2018..1220d34b6 100644 --- a/test/size-ledger/ship/gsta-167-pr34.json +++ b/test/size-ledger/ship/gsta-167-pr34.json @@ -1,4 +1,4 @@ { - "bytes": 1, - "why": "gstack-ship-preflight skips the base merge under queue_mode stamp-at-merge (Step 3 note)" + "bytes": 1371, + "why": "GSTA-167 #3633 meter fixes: own-jam meter, reservation id line and every capped start on --reservation, Red Team takes no start, docs-audit settle in its dispatch program, step priors from the run's median per-call charge" }