From f30c33f7d58ab347b99ca506dbed75f344e69851 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 16:28:27 +0000 Subject: [PATCH] fix(ship-budget): calibrate projections from the run's own meter, settle dispatch reservations when reviewers return (GST-101) The /ship budget projected every step and dispatch from fixed constants (7.43M per dispatch, 2.25M per default step) plus a fixed 4.48M reserve, so real caps refused work that cost 3-20x less (GSTA-152 PR B, GSTA-2). Each settled reservation is now measured on the next reading, and the run projects with its own factor (max actual/prior of its last three measurements x 1.5), the prior only before the first measurement. The default tail reserve scales with the step factor; the stop overhead keeps its fit. Reservations of reviewers admitted one at a time no longer pile up: check --returned settles named dispatches, --finish settles the pass's dispatches of that step, and a step change settles consumed dispatches. DRIFT and BUDGET_DRIFT lines print projected against actual. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-review-log | 10 +- bin/gstack-ship-budget | 4 +- docs/ship-budget.md | 31 ++- lib/ship-budget-cli.ts | 60 ++++-- lib/ship-budget.ts | 76 ++++++- ship/sections/budget-handoff.md | 3 +- ship/sections/budget-handoff.md.tmpl | 3 +- test/ship-budget-projection.test.ts | 199 ++++++++++++++++++ test/ship-budget.test.ts | 12 +- .../ship/gst-101-ship-budget-calibration.json | 4 + 10 files changed, 357 insertions(+), 45 deletions(-) create mode 100644 test/ship-budget-projection.test.ts create mode 100644 test/size-ledger/ship/gst-101-ship-budget-calibration.json diff --git a/bin/gstack-review-log b/bin/gstack-review-log index 0a7037120..ae60e73a7 100755 --- a/bin/gstack-review-log +++ b/bin/gstack-review-log @@ -6,7 +6,8 @@ # Under a /ship token cap (docs/ship-budget.md): --start SKILL --reservation ID # with the RESERVATION id gstack-ship-budget check printed for this dispatch # batch; refuses (exit 1, BUDGET_CHECK_SKIPPED) without an unspent one, and -# --finish settles it. Without a cap no marker exists and nothing changes. +# --finish settles it with the pass's other dispatch reservations of that +# step. Without a cap no marker exists and nothing changes. # Fix-range repeat (#3339): gstack-review-log --start review --fix-range # prints the token, then FIX_BASE=; refuses (exit 1) unless the latest # pass completed with fixes on unchanged content and its commit is an @@ -151,12 +152,13 @@ try { echo "$STAMPED" >> "$GSTACK_HOME/projects/$SLUG/$BRANCH-reviews.jsonl" -# A finished dispatch settles the budget reservation its start consumed (its -# completion receipt); stdout stays unchanged and a failure never undoes the row. +# A finished pass settles the budget reservation its start consumed (its +# completion receipt) and every other dispatch reservation started in the same +# step; stdout stays unchanged and a failure never undoes the row. if [ -n "$TOKEN" ] && [ -f "$GSTACK_SHIP_BUDGET_MARKER" ]; then GSTACK_REVIEW_TOKEN="$TOKEN" bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e ' const { settleReviewFinish } = await import(process.env.GSTACK_SHIP_BUDGET_LIB); - try { const id = settleReviewFinish(process.env.GSTACK_SHIP_BUDGET_MARKER, process.env.GSTACK_REVIEW_TOKEN); if (id) console.error("review-log: settled budget reservation " + id); } + try { const id = settleReviewFinish(process.env.GSTACK_SHIP_BUDGET_MARKER, process.env.GSTACK_REVIEW_TOKEN); if (id) console.error("review-log: settled budget reservations " + id); } catch (e) { console.error("review-log: budget reservation not settled: " + e.message); } ' || true fi diff --git a/bin/gstack-ship-budget b/bin/gstack-ship-budget index ecb3277dc..3f1361c0e 100755 --- a/bin/gstack-ship-budget +++ b/bin/gstack-ship-budget @@ -7,8 +7,8 @@ * * start --record [--token-cap ] [--meter capy|claude|estimate] [--reading ] [--prior-reading ] * [--session |--nonce ] [--reserve ] [--delegate auto|never] [--calibration ] [--ledger ] - * check --run |--record --step --next [,...]> [--reading ] [--prior-reading ] - * settle --run|--record --reservation [--unknown] release --run|--record --reservation + * check --run |--record --step --next [,...]> [--reading ] [--prior-reading ] [--returned [,...]] + * settle --run|--record --reservation [--unknown] [--reading ] release --run|--record --reservation * handoff --run|--record --from --to --base origin/ [--reviewed-tree ] [--trail --trail-run ] * [--sub-cap ] [--reading ] [--child-session ] * launch --in (Claude Code: background headless child with explicit permissions) diff --git a/docs/ship-budget.md b/docs/ship-budget.md index ef7558364..ef2bdd7f0 100644 --- a/docs/ship-budget.md +++ b/docs/ship-budget.md @@ -17,7 +17,9 @@ Exactly at the cap is admitted; one token over is not. A parallel review batch i one sum before any of it starts. Admitted work can still cost more than its projection, and gstack cannot stop a host mid-turn from bash, so a run can finish above the cap by the misprojection of work already admitted. The tail reserve (default 2M, `--reserve `) keeps -room for the receipt steps, and the stop overhead (calibrated) keeps room to stop cleanly. +room for the receipt steps, and the stop overhead (calibrated) keeps room to stop cleanly. Once +the run has measured a step, the tail reserve scales with its measured step cost (a `--reserve` +you set stays as given); the stop overhead never scales. ## Quickstart @@ -48,7 +50,7 @@ With a cap set and no readable meter, start refuses ## The BUDGET line - BUDGET: used= reserved= unknown= admission_cap= tail= unit= meter= epoch= call= next= projected= verdict=continue|delegate|stop + BUDGET: used= reserved= unknown= admission_cap= tail= unit= meter= epoch= call= next= projected= calibration=step:,dispatch: verdict=continue|delegate|stop | Field | Meaning | |---|---| @@ -56,10 +58,11 @@ With a cap set and no readable meter, start refuses | `reserved` | Open reservations: admitted steps, started dispatches, and each open child's sub-cap counted as max(sub_cap, child_used), never both | | `unknown` | Charges the meter cannot see (an external-CLI outside voice on Claude Code), counted at their reservation | | `admission_cap` | The cap, or `none` | -| `tail` | Tail reserve plus stop overhead, held back from every admission | +| `tail` | Tail reserve (scaled by the step factor once measured) plus stop overhead, held back from every admission | | `unit`, `meter`, `epoch` | The meter contract: `capy-tokens` (root counter), `claude-usage-tokens` (journal usage), `estimate-tokens`; a meter change opens a new epoch | | `call` | The per-call charge: two Capy readings in consecutive calls, or the last Claude Code call's usage | -| `next`, `projected` | What the check was for and its calibrated projection | +| `next`, `projected` | What the check was for and its projection: the prior times the run's factor for that kind | +| `calibration` | The run's step and dispatch factors (see Calibration), or `prior` before the first measurement | | `verdict` | `continue` (exit 0, prints `RESERVATION:` under a cap), `delegate` (exit 0), `stop` (exit 3) | Any exit other than 0 means the next step does not start. Exit 2 is a usage or validation @@ -77,7 +80,12 @@ gstack-review-log --start review --reservation ``` Roles are review-depth's: `specialist`, `red-team`, `adversarial-native`, `outside-voice`, -`fix-check`. `--finish ` settles the reservation at the dispatch's completion receipt. +`fix-check`. `--finish ` is the pass's completion receipt: it settles that reservation and +every other dispatch reservation started in the same step, so reviewers admitted one at a time +do not stay reserved. Name a reviewer that came back in your next check +(`check ... --returned [,...]`), or, when other work comes first, `settle --reservation +--reading ` as it returns so its cost is measured alone. A check that moves to the next step +settles every dispatch a reviewer started in the step before. `close` audits after the fact: each review-log row of the run with no reservation gets one BUDGET_CHECK_SKIPPED line and the run ends status=incomplete for the owner to accept (`close --approved-by --words ""`); a reservation is never created after the fact. @@ -137,6 +145,19 @@ across three runs; per-dispatch has two observations and stop overhead one. Only meter has observations; the Claude Code and estimate units are seeded from it until a Claude-metered /ship completes. +These constants are the prior. Each settled step or dispatch is measured on the next fresh +reading (a step from its check's reading to the next one; dispatches settled together as one +group, split by role), and prints + + DRIFT: id= kind=step|dispatch next= projected= actual= ratio= + +From then on the run projects steps and dispatches with its own factor: the largest +actual/prior ratio among its last three measurements of that kind, times 1.5. The step factor +also scales the default tail reserve. Dispatches keep the prior until the first +dispatch is measured, so under a tight cap admit one reviewer first and let it calibrate the rest. +`close` prints `BUDGET_DRIFT:` with the run's projected and actual totals per kind. The estimate +meter has no readings, so it always projects at the prior. + ## Token cap and USD spend cap A lane's `--spend-cap` (USD) bounds paid gates and evals through lib/spend-ledger.ts; `--token-cap` diff --git a/lib/ship-budget-cli.ts b/lib/ship-budget-cli.ts index d761c2e74..55f83e357 100644 --- a/lib/ship-budget-cli.ts +++ b/lib/ship-budget-cli.ts @@ -13,9 +13,9 @@ import * as path from 'node:path'; import { resultLine } from './result-codes'; import { BudgetError, DEFAULT_TAIL, DELEGATE_SEGMENT_CALL, DELEGATE_START_CALL, METER_CONTRACTS, SEGMENTS, SHIP_STEPS, - admits, applyReading, budgetError, budgetLine, ledgerFileFor, loadCalibration, mintRunId, mutateLedger, newEpoch, - parseNext, parseRecordBlock, parseStep, parseTokens, pointMarker, readLedger, reserveIn, restoreRun, reviewPaths, - runDirFor, segmentEstimate, settleIn, stepProjection, projection, tally, unitConstants, unreservedDispatches, usageError, writeLedger, + admits, applyReading, budgetError, budgetLine, ledgerFileFor, loadCalibration, measureSettled, mintRunId, mutateLedger, newEpoch, + parseNext, parseRecordBlock, parseStep, parseTokens, pointMarker, projector, readLedger, reserveIn, restoreRun, reviewPaths, + runDirFor, settleIn, stepProjection, tally, unitConstants, unreservedDispatches, usageError, writeLedger, type Ledger, type MeterKind, type Next, } from './ship-budget'; import { claudeConfigDir, loadMeterState, newMeterState, nonceSessions, readClaudeUsage, saveMeterState, sessionJournals } from './claude-usage'; @@ -41,6 +41,8 @@ export function parseArgs(argv: string[], values: string[], flags: string[] = [] export const str = (o: Opts, k: string) => (typeof o[k] === 'string' ? (o[k] as string) : undefined); export const tokensOpt = (o: Opts, k: string) => (str(o, k) === undefined ? undefined : parseTokens(str(o, k), `--${k}`)); const meterFile = (l: Ledger) => path.join(l.dir, 'claude-meter.json'); +/** Whether this command took a fresh reading (Claude Code reads its journals on every command). */ +const freshReading = (l: Ledger, o: Opts) => l.meter.contract.meter === 'claude' || str(o, 'reading') !== undefined; /** Take this command's reading under the run's meter contract; returns the per-call charge when known. */ export function readMeter(l: Ledger, o: Opts, requireReading: boolean): number | null { @@ -69,13 +71,13 @@ export function readMeter(l: Ledger, o: Opts, requireReading: boolean): number | export function stopLines(l: Ledger, f: { call: number | null; next: string; step: string; projected: number; reason: 'budget_exhausted' | 'context_threshold' }): string[] { l.stopped = { at: new Date().toISOString(), reason: f.reason, step: f.step, next: f.next }; const t = tally(l); - const cal = loadCalibration(l.calibration); + const p = projector(l); const from = f.next.startsWith('dispatch:') ? f.step : f.next; - const tailCost = segmentEstimate(cal, l.meter.contract.unit, from, '22') + l.tail + l.stop_overhead; + const tailCost = p.segment(from, '22') + p.reserve; const lines = [budgetLine(l, { ...f, verdict: 'stop' })]; if (f.reason === 'budget_exhausted') { lines.push(`STOP: reason=budget_exhausted spent=${t.used} reserved=${t.reserved} unknown=${t.unknown} admission_cap=${l.cap} next=${f.next} projected=${f.projected}`, - resultLine('gstack-ship-budget', 'BUDGET_CAP_REACHED', `next ${f.next} needs ${f.projected} + reserve ${l.tail + l.stop_overhead}`), + resultLine('gstack-ship-budget', 'BUDGET_CAP_REACHED', `next ${f.next} needs ${f.projected} + reserve ${p.reserve}`), `RECOVER: /ship --resume ${l.record ?? ''} --token-cap ${t.used + t.reserved + t.unknown + tailCost}`); } else { lines.push(`STOP: reason=context_threshold call=${f.call ?? 'unknown'} threshold=${DELEGATE_SEGMENT_CALL} next=${f.next} (${l.role === 'child' ? 'a child never delegates' : '--delegate never'})`, @@ -93,10 +95,10 @@ export function writeReturnFile(file: string, lines: string[]): void { /** Admission and the delegate rule for one boundary; reserves the admitted work under a cap. */ export function decide(l: Ledger, step: string, next: Next, call: number | null, startThreshold = false): Outcome { - const cal = loadCalibration(l.calibration); + const p = projector(l); const nextId = next.kind === 'step' ? next.step : `dispatch:${next.roles.join(',')}`; - const projected = projection(cal, l.meter.contract.unit, next); - if (!admits(tally(l), projected, l.tail + l.stop_overhead, l.cap)) { + const { prior, tokens: projected } = p.project(next); + if (!admits(tally(l), projected, p.reserve, l.cap)) { return { stdout: stopLines(l, { call, next: nextId, step, projected, reason: 'budget_exhausted' }), exit: 3 }; } const boundary = next.kind === 'step' && (startThreshold ? next.step === '1' : next.step === '7' || next.step === '12'); @@ -110,7 +112,7 @@ export function decide(l: Ledger, step: string, next: Next, call: number | null, } const lines = [budgetLine(l, { call, next: nextId, projected, verdict: 'continue' })]; if (l.cap !== null) { - const r = reserveIn(l, { kind: next.kind, step, next: nextId, tokens: projected, ...(next.kind === 'dispatch' ? { roles: next.roles } : {}) }); + const r = reserveIn(l, { kind: next.kind, step, next: nextId, tokens: projected, prior, ...(next.kind === 'dispatch' ? { roles: next.roles } : {}) }); lines.push(`RESERVATION: id=${r.id} next=${nextId} tokens=${projected}`); } return { stdout: lines, exit: 0 }; @@ -166,7 +168,8 @@ export function cmdStart(argv: string[], ctx: Ctx): Outcome { const sessions = meter === 'claude' ? bindClaude(o, ctx.env) : undefined; const l: Ledger = { schema_version: 1, kind: 'gstack-ship-budget', run, dir, file: str(o, 'ledger') ?? path.join(dir, 'budget.json'), record: path.resolve(record), - created_at: new Date().toISOString(), cap, tail: tokensOpt(o, 'reserve') ?? DEFAULT_TAIL, stop_overhead: unitConstants(cal, contract.unit).stop_overhead, + created_at: new Date().toISOString(), cap, tail: tokensOpt(o, 'reserve') ?? DEFAULT_TAIL, ...(tokensOpt(o, 'reserve') !== undefined ? { tail_explicit: true } : {}), + stop_overhead: unitConstants(cal, contract.unit).stop_overhead, delegate, role: 'parent', meter: { contract, epoch: 1, baseline: 0, last: null, ...(sessions ? { sessions, config_dir: claudeConfigDir(ctx.env) } : {}) }, epochs: [], carried_used: 0, estimate_used: 0, unknown: 0, next_id: 1, reservations: [], handoffs: [], cap_raises: [], marker: paths.marker, review_log: paths.reviewLog, ...(str(o, 'calibration') ? { calibration: path.resolve(str(o, 'calibration')!) } : {}), @@ -181,40 +184,59 @@ export function cmdStart(argv: string[], ctx: Ctx): Outcome { return { stdout: [`BUDGET_RUN: id=${run} ledger=${l.file}`, ...out.stdout], exit: out.exit }; } +/** + * check: read the meter, settle what is done (open step reservations, the + * dispatches named by --returned, and on a step change every dispatch a + * reviewer consumed), measure them, then admit the next step or dispatch. + */ export function cmdCheck(argv: string[], ctx: Ctx): Outcome { - const o = parseArgs(argv, ['run', 'record', 'step', 'next', 'reading', 'prior-reading']); + const o = parseArgs(argv, ['run', 'record', 'step', 'next', 'reading', 'prior-reading', 'returned']); const step = parseStep(str(o, 'step')); const next = parseNext(str(o, 'next')); + const returned = (str(o, 'returned') ?? '').split(',').filter(Boolean); return mutateLedger(ledgerFileFor({ run: str(o, 'run'), record: str(o, 'record') }, ctx.env), ctx.env, l => { if (l.closed_at) throw usageError(`run ${l.run} is closed`); if (l.stopped) return { stdout: [budgetLine(l, { call: null, next: l.stopped.next, projected: 0, verdict: 'stop' }), `STOP: reason=${l.stopped.reason} at Step ${l.stopped.step}; resume first: gstack-ship-budget resume --record ${l.record}`, `GSTACK_RESULT: skill=ship status=incomplete run=${l.dir}`], exit: 3 }; - for (const r of l.reservations) if (r.kind === 'step' && r.state === 'open') settleIn(l, r); + const back = returned.map(id => l.reservations.find(r => r.id === id && r.kind === 'dispatch')); + const bad = returned.filter((_, i) => !back[i]); + if (bad.length) throw usageError(`--returned ${bad.join(',')} is not a dispatch reservation of run ${l.run}; fix: pass the RESERVATION ids check printed for the reviewers that came back`); const call = readMeter(l, o, true); - return decide(l, step, next, call); + for (const r of l.reservations) { + const done = r.kind === 'step' ? r.state === 'open' : r.kind === 'dispatch' && (back.includes(r) || (next.kind === 'step' && r.state === 'consumed')); + if (done) settleIn(l, r); + } + const drift = freshReading(l, o) ? measureSettled(l) : []; + const out = decide(l, step, next, call); + return { stdout: [...drift, ...out.stdout], exit: out.exit }; }); } export function cmdSettle(argv: string[], ctx: Ctx, release = false): Outcome { - const o = parseArgs(argv, ['run', 'record', 'reservation'], release ? [] : ['unknown']); + const o = parseArgs(argv, ['run', 'record', 'reservation', ...(release ? [] : ['reading'])], release ? [] : ['unknown']); const id = str(o, 'reservation'); if (!id) throw usageError(`${release ? 'release' : 'settle'} needs --reservation `); return mutateLedger(ledgerFileFor({ run: str(o, 'run'), record: str(o, 'record') }, ctx.env), ctx.env, l => { const r = l.reservations.find(x => x.id === id); if (!r) throw usageError(`reservation ${id} is not in run ${l.run}`); - if (release) { if (r.state === 'open') r.state = 'released'; } - else settleIn(l, r, { unknown: o.unknown === true, cal: loadCalibration(l.calibration) }); - return { stdout: [`${release ? 'RELEASED' : 'SETTLED'}: id=${id} state=${r.state}`], exit: 0 }; + if (release) { if (r.state === 'open') r.state = 'released'; return { stdout: [`RELEASED: id=${id} state=${r.state}`], exit: 0 }; } + readMeter(l, o, false); + settleIn(l, r, { unknown: o.unknown === true, cal: loadCalibration(l.calibration) }); + return { stdout: [`SETTLED: id=${id} state=${r.state}`, ...(freshReading(l, o) ? measureSettled(l) : [])], exit: 0 }; }); } export function cmdClose(argv: string[], ctx: Ctx): Outcome { const o = parseArgs(argv, ['run', 'record', 'reading', 'approved-by', 'words']); return mutateLedger(ledgerFileFor({ run: str(o, 'run'), record: str(o, 'record') }, ctx.env), ctx.env, l => { - for (const r of l.reservations) if (r.kind === 'step' && r.state === 'open') settleIn(l, r); readMeter(l, o, false); + for (const r of l.reservations) if (r.kind === 'step' && r.state === 'open') settleIn(l, r); + if (freshReading(l, o)) measureSettled(l); const t = tally(l); const open = l.reservations.filter(r => r.kind === 'dispatch' && r.state === 'consumed').length; const lines = [`BUDGET_FINAL: used=${t.used} reserved=${t.reserved} unknown=${t.unknown} admission_cap=${l.cap ?? 'none'} unit=${l.meter.contract.unit} meter=${l.meter.contract.meter} epoch=${l.meter.epoch} open_dispatches=${open}`]; + const sum = (kind: string, k: 'projected' | 'actual') => (l.samples ?? []).filter(s => s.kind === kind).reduce((n, s) => n + s[k], 0); + const count = (kind: string) => (l.samples ?? []).filter(s => s.kind === kind).length; + lines.push(`BUDGET_DRIFT: steps=${count('step')} step_projected=${sum('step', 'projected')} step_actual=${sum('step', 'actual')} dispatches=${count('dispatch')} dispatch_projected=${sum('dispatch', 'projected')} dispatch_actual=${sum('dispatch', 'actual')}`); const accepted = new Set((l as any).accepted_skips as string[] | undefined ?? []); const skipped = l.cap === null ? [] : unreservedDispatches(l).filter(s => !accepted.has(s.started)); if (skipped.length && str(o, 'approved-by') && str(o, 'words')) { diff --git a/lib/ship-budget.ts b/lib/ship-budget.ts index a3c3d18e1..6768a01ad 100644 --- a/lib/ship-budget.ts +++ b/lib/ship-budget.ts @@ -51,7 +51,9 @@ export const METER_CONTRACTS: Record = { export interface Reservation { id: string; kind: 'step' | 'dispatch' | 'child'; step: string; next: string; roles?: string[]; tokens: number; at: string; state: 'open' | 'consumed' | 'settled' | 'released'; token?: string; started_at?: string; settled_at?: string; + prior?: number; reading_at?: number | null; epoch?: number; actual?: number; } +export interface Sample { id: string; kind: 'step' | 'dispatch'; epoch: number; prior: number; projected: number; actual: number } export interface Handoff { n: number; sha: string; file: string; from: string; to: string; sub_cap: number; reservation: string; reading_at: number | null; epoch: number; return_file: string; reconciled_at?: string; status?: string; child_used?: number; @@ -65,7 +67,7 @@ export interface Ledger { carried_used: number; estimate_used: number; unknown: number; next_id: number; reservations: Reservation[]; handoffs: Handoff[]; cap_raises: Array<{ from: number | null; to: number | null; by: string; words: string; at: string }>; - marker?: string; review_log?: string; calibration?: string; + marker?: string; review_log?: string; calibration?: string; samples?: Sample[]; tail_explicit?: boolean; stopped?: { at: string; reason: string; step: string; next: string }; closed_at?: string; } @@ -134,7 +136,61 @@ export function projection(cal: Calibration, unit: string, next: Next): number { return next.kind === 'dispatch' ? next.roles.length * unitConstants(cal, unit).per_dispatch : stepProjection(cal, unit, next.step); } -/** Steps from..to inclusive plus their usual reviewer batches: a child's segment estimate and the RECOVER tail. */ +/** How many recent measurements of a kind the run's factor looks at (GST-101). */ +export const SAMPLE_WINDOW = 3; +const roundUp = (n: number, to: number) => Math.ceil(n / to) * to; + +/** + * The run's own factor for steps or dispatches: the largest actual/prior ratio + * among its last SAMPLE_WINDOW measurements times the calibration headroom, or + * null before the first measurement (the prior applies until then). + */ +export function measuredFactor(l: Ledger, cal: Calibration, kind: Sample['kind']): number | null { + const rows = (l.samples ?? []).filter(s => s.kind === kind && s.epoch === l.meter.epoch && s.actual > 0 && s.prior > 0).slice(-SAMPLE_WINDOW); + return rows.length ? Math.max(...rows.map(s => s.actual / s.prior)) * cal.headroom : null; +} + +/** + * Projections calibrated by the run's measured factors. The tail reserve scales + * with the step factor unless --reserve set it; the stop overhead keeps its + * calibrated fit so a run always has room to stop. + */ +export function projector(l: Ledger, cal: Calibration = loadCalibration(l.calibration)) { + const unit = l.meter.contract.unit; + const step = measuredFactor(l, cal, 'step'), dispatch = measuredFactor(l, cal, 'dispatch'); + const scale = (n: number, f: number | null) => (f === null ? n : roundUp(n * f, cal.round_to)); + const project = (next: Next) => { + const prior = projection(cal, unit, next); + return { prior, tokens: scale(prior, next.kind === 'dispatch' ? dispatch : step) }; + }; + const segment = (from: string, to: string) => SHIP_STEPS.slice(SHIP_STEPS.indexOf(from as any), SHIP_STEPS.indexOf(to as any) + 1) + .reduce((sum, s) => sum + project({ kind: 'step', step: s }).tokens + (cal.segment_dispatches[s] ?? 0) * project({ kind: 'dispatch', roles: ['specialist'] }).tokens, 0); + const label = (f: number | null) => (f === null ? 'prior' : f.toFixed(3)); + return { project, segment, reserve: (l.tail_explicit ? l.tail : scale(l.tail, step)) + l.stop_overhead, label: `step:${label(step)},dispatch:${label(dispatch)}` }; +} + +/** + * Measure settled reservations against the current (fresh) reading: a step + * from its reading to now; settled dispatches as one group from the earliest + * of their readings, split by role. Returns one DRIFT line per measurement. + */ +export function measureSettled(l: Ledger): string[] { + const last = l.meter.last; + if (last === null) return []; + const pending = l.reservations.filter(r => r.state === 'settled' && r.kind !== 'child' && r.actual === undefined + && typeof r.reading_at === 'number' && r.epoch === l.meter.epoch && last >= r.reading_at); + const dispatches = pending.filter(r => r.kind === 'dispatch'); + const from = Math.min(...dispatches.map(r => r.reading_at!)); + const roles = dispatches.reduce((n, r) => n + (r.roles?.length ?? 1), 0); + return pending.map(r => { + const actual = r.kind === 'dispatch' ? Math.round(((last - from) * (r.roles?.length ?? 1)) / roles) : last - r.reading_at!; + r.actual = actual; + (l.samples ??= []).push({ id: r.id, kind: r.kind as Sample['kind'], epoch: l.meter.epoch, prior: r.prior ?? r.tokens, projected: r.tokens, actual }); + return `DRIFT: id=${r.id} kind=${r.kind} next=${r.next} projected=${r.tokens} actual=${actual} ratio=${(actual / Math.max(1, r.tokens)).toFixed(2)}`; + }); +} + +/** Steps from..to inclusive plus their usual reviewer batches at the prior: a child's segment estimate. */ export function segmentEstimate(cal: Calibration, unit: string, from: string, to: string): number { const i = SHIP_STEPS.indexOf(from as any), j = SHIP_STEPS.indexOf(to as any); const k = unitConstants(cal, unit); @@ -206,11 +262,12 @@ export function admits(t: { used: number; reserved: number; unknown: number }, p export function budgetLine(l: Ledger, f: { call: number | null; next: string; projected: number; verdict: string }): string { const t = tally(l); - return `BUDGET: used=${t.used} reserved=${t.reserved} unknown=${t.unknown} admission_cap=${l.cap ?? 'none'} tail=${l.tail + l.stop_overhead} unit=${l.meter.contract.unit} meter=${l.meter.contract.meter} epoch=${l.meter.epoch} call=${f.call ?? 'unknown'} next=${f.next} projected=${f.projected} verdict=${f.verdict}`; + const p = projector(l); + return `BUDGET: used=${t.used} reserved=${t.reserved} unknown=${t.unknown} admission_cap=${l.cap ?? 'none'} tail=${p.reserve} unit=${l.meter.contract.unit} meter=${l.meter.contract.meter} epoch=${l.meter.epoch} call=${f.call ?? 'unknown'} next=${f.next} projected=${f.projected} calibration=${p.label} verdict=${f.verdict}`; } export function reserveIn(l: Ledger, r: Omit): Reservation { - const row: Reservation = { ...r, id: `${l.run}-r${l.next_id++}`, at: new Date().toISOString(), state: 'open' }; + const row: Reservation = { ...r, id: `${l.run}-r${l.next_id++}`, at: new Date().toISOString(), state: 'open', reading_at: l.meter.last, epoch: l.meter.epoch }; l.reservations.push(row); return row; } @@ -333,15 +390,20 @@ export function admitReviewStart(marker: string, reservation: string | undefined }); } -/** gstack-review-log --finish TOKEN: settle the reservation that start consumed (its completion receipt). */ +/** + * gstack-review-log --finish TOKEN: the pass is over, so settle the reservation + * that start consumed and every other dispatch reservation started in the same + * step (a pass logs one row even when its reviewers were admitted one by one). + */ export function settleReviewFinish(marker: string, token: string, env: NodeJS.ProcessEnv = process.env): string | undefined { const active = activeLedger(marker); if (!active || !active.reservations.some(r => r.token === token && r.state === 'consumed')) return undefined; return mutateLedger(active.file, env, l => { const r = l.reservations.find(x => x.token === token && x.state === 'consumed'); if (!r) return undefined; - settleIn(l, r); - return r.id; + const pass = l.reservations.filter(x => x.kind === 'dispatch' && x.state === 'consumed' && x.step === r.step); + for (const x of pass) settleIn(l, x); + return pass.map(x => x.id).join(','); }); } diff --git a/ship/sections/budget-handoff.md b/ship/sections/budget-handoff.md index 1b036c551..5529caca6 100644 --- a/ship/sections/budget-handoff.md +++ b/ship/sections/budget-handoff.md @@ -25,7 +25,8 @@ Before each dispatch batch, reserve the batch as one sum, then pass the printed ~/.claude/skills/gstack/bin/gstack-review-log --start review --reservation "$RESERVATION_ID" ``` -`--finish` settles it. A dispatch with no `review-log --start` settles with +`--finish` settles it and the pass's other dispatch reservations. Admitting reviewers one at +a time, name each one that came back in the next check's `--returned `. A dispatch with no `review-log --start` settles with `gstack-ship-budget settle --run --reservation ` when it returns (`--unknown` when the meter cannot see it), or `release` if it never ran. diff --git a/ship/sections/budget-handoff.md.tmpl b/ship/sections/budget-handoff.md.tmpl index fa41ceb04..30fff488e 100644 --- a/ship/sections/budget-handoff.md.tmpl +++ b/ship/sections/budget-handoff.md.tmpl @@ -23,7 +23,8 @@ Before each dispatch batch, reserve the batch as one sum, then pass the printed ~/.claude/skills/gstack/bin/gstack-review-log --start review --reservation "$RESERVATION_ID" ``` -`--finish` settles it. A dispatch with no `review-log --start` settles with +`--finish` settles it and the pass's other dispatch reservations. Admitting reviewers one at +a time, name each one that came back in the next check's `--returned `. A dispatch with no `review-log --start` settles with `gstack-ship-budget settle --run --reservation ` when it returns (`--unknown` when the meter cannot see it), or `release` if it never ran. diff --git a/test/ship-budget-projection.test.ts b/test/ship-budget-projection.test.ts new file mode 100644 index 000000000..940ae169e --- /dev/null +++ b/test/ship-budget-projection.test.ts @@ -0,0 +1,199 @@ +/** + * GST-101: the /ship budget projection is calibrated from the run's own meter + * (the prior only until the first measurement), the tail reserve scales with + * it, a dispatch's reservation settles when its reviewer returns, and each + * measured reservation prints its projected and actual cost. The replays use + * the numbers measured on GSTA-2 (gstack-evals #35/#36, 1.91.319.0) and on + * GSTA-152 PR B (#3596, run ship-20261029T083816-a4cc20; handoffs/token-efficiency/ + * ship-budget-projection-GSTA-152-prb.md in the GStack Minder drive). + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { loadCalibration, SHIP_STEPS, unitConstants } from '../lib/ship-budget'; +import { budgetHarness, type Harness } from './helpers/ship-budget-harness'; + +const cal = loadCalibration(); +const K = unitConstants(cal, 'capy-tokens'); +const harnesses: Harness[] = []; +const fresh = () => { const h = budgetHarness(); harnesses.push(h); return h; }; +afterEach(() => { while (harnesses.length) harnesses.pop()!.cleanup(); }); + +function started(h: Harness, cap: string) { + const r = h.sb(['start', '--record', h.record(), '--token-cap', cap, '--reading', '0']); + expect(r.code).toBe(0); + return { run: h.field(r.out, 'BUDGET_RUN', 'id'), ledger: h.field(r.out, 'BUDGET_RUN', 'ledger') }; +} +const finishRow = () => JSON.stringify({ skill: 'review', timestamp: new Date().toISOString(), status: 'clean', issues_found: 0, critical: 0, informational: 0 }); + +/** GSTA-2: about 0.27M per step on Steps 1-5 (1.35M), 165-200k per call; the prior projected 2.25M per step plus a 4.48M reserve. */ +const GSTA2_STEP = 270_000; +const GSTA2_STOP = 4 * 200_000; +/** GSTA-152 PR B pass 2, one reviewer at a time on a 127-line fix range (counter delta per dispatch). */ +const PASS2: Array<[string, number]> = [ + ['specialist', 373_254], ['specialist', 511_454], ['specialist', 364_113], ['specialist', 362_494], + ['specialist', 366_058], ['specialist', 234_661], ['red-team', 1_016_168], +]; +const PASS3: Array<[string, number]> = [['red-team', 1_049_900], ['fix-check', 368_801]]; +/** One parent call late in GSTA-152 (376-384k), and #26's measured stop overhead. */ +const PARENT_CALL = 380_000; +const LEAN_DISPATCH = 500_000; +const STOP_26 = 1_650_000; + +describe('projection calibrated from the run\'s own meter (GST-101)', () => { + test('GSTA-2 replay: an 8M cap admits Steps 6 and 6.5 after Steps 1-5 cost 0.27M each, and stops with room to stop', () => { + const h = fresh(); + const cap = 8_000_000; + const s = started(h, '8M'); + const steps = SHIP_STEPS.slice(SHIP_STEPS.indexOf('1')); + let reading = 0; + const admitted: string[] = []; + let stop = ''; + for (let i = 0; i + 1 < steps.length; i++) { + const calls = cal.step_calls[steps[i]!] ?? cal.step_calls.default!; + reading += Math.round((GSTA2_STEP / 3) * calls * (1 + 0.05 * i)) + (cal.step_dispatches[steps[i]!] ?? 0) * LEAN_DISPATCH; + expect(reading, `spend after Step ${steps[i]}`).toBeLessThanOrEqual(cap); + const c = h.sb(['check', '--run', s.run, '--step', steps[i]!, '--next', steps[i + 1]!, '--reading', String(reading)]); + if (c.code === 3) { stop = c.out; break; } + expect(c.code).toBe(0); + admitted.push(steps[i + 1]!); + if (steps[i + 1] === '6') expect(Number(h.field(c.out, 'BUDGET', 'projected'))).toBeLessThan(1_000_000); + } + expect(admitted).toEqual(['2', '3', '4', '5', '6', '6.5']); + expect(stop).toContain('STOP: reason=budget_exhausted'); + expect(reading + GSTA2_STOP).toBeLessThanOrEqual(cap); + }); + + test('GSTA-152 PR B replay: per-dispatch admission never holds returned reviewers, the 29.28M refusal is admitted, and the run stops before the cap', () => { + const h = fresh(); + const cap = 29_280_000; + const s = started(h, '29.28M'); + let reading = 0; + const steps = SHIP_STEPS.slice(SHIP_STEPS.indexOf('1'), SHIP_STEPS.indexOf('9') + 1); + for (let i = 0; i + 1 < steps.length; i++) { + reading += 750_000; + expect(h.sb(['check', '--run', s.run, '--step', steps[i]!, '--next', steps[i + 1]!, '--reading', String(reading)]).code).toBe(0); + } + let previous = ''; + const dispatch = (role: string) => { + reading += PARENT_CALL; + const c = h.sb(['check', '--run', s.run, '--step', '9', '--next', `dispatch:${role}`, '--reading', String(reading), ...(previous ? ['--returned', previous] : [])]); + return c; + }; + for (const [role, cost] of [...PASS2, ...PASS3]) { + const c = dispatch(role); + expect(c.code, c.out + c.err).toBe(0); + previous = h.field(c.out, 'RESERVATION', 'id'); + expect(h.reviewLog(['--start', 'review', '--reservation', previous]).code).toBe(0); + reading += cost; + } + const lean = h.ledger(s.ledger).reservations.filter((r: any) => r.kind === 'dispatch' && r.actual !== undefined); + expect(lean.length).toBe(PASS2.length + PASS3.length - 1); + const settled = h.sb(['settle', '--run', s.run, '--reservation', previous, '--reading', String(reading)]); + expect(settled.out).toMatch(/^DRIFT: id=\S+ kind=dispatch next=dispatch:fix-check projected=\d+ actual=368801 ratio=\d+\.\d\d$/m); + reading = 17_790_000; + previous = ''; + const fixCheck = dispatch('fix-check'); + expect(fixCheck.code, fixCheck.out).toBe(0); + const projected = Number(h.field(fixCheck.out, 'BUDGET', 'projected')); + expect(projected).toBeLessThan(K.per_dispatch / 3); + expect(projected).toBeGreaterThanOrEqual(1_049_900); + previous = h.field(fixCheck.out, 'RESERVATION', 'id'); + expect(h.reviewLog(['--start', 'review-fix-check', '--reservation', previous]).code).toBe(0); + reading += 368_801; + let parent = PARENT_CALL; + let stop = ''; + for (let round = 0; round < 40 && !stop; round++) { + parent += 50_000; + reading += parent; + const c = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:fix-check', '--reading', String(reading), '--returned', previous]); + if (c.code === 3) { stop = c.out; break; } + previous = h.field(c.out, 'RESERVATION', 'id'); + expect(h.reviewLog(['--start', 'review-fix-check', '--reservation', previous]).code).toBe(0); + reading += 368_801; + expect(reading).toBeLessThanOrEqual(cap); + } + expect(stop).toContain('STOP: reason=budget_exhausted'); + expect(reading + STOP_26).toBeLessThanOrEqual(cap); + }); + + test('the prior holds until the first measurement; the tail reserve scales with the measured step cost', () => { + const h = fresh(); + const s = started(h, '100M'); + const first = h.sb(['check', '--run', s.run, '--step', '1', '--next', '2', '--reading', '0']); + expect(Number(h.field(first.out, 'BUDGET', 'projected'))).toBe(3 * K.per_call); + expect(Number(h.field(first.out, 'BUDGET', 'tail'))).toBe(2_000_000 + K.stop_overhead); + expect(h.field(first.out, 'BUDGET', 'calibration')).toBe('step:prior,dispatch:prior'); + const second = h.sb(['check', '--run', s.run, '--step', '2', '--next', '3', '--reading', '450000']); + const factor = (450_000 / (3 * K.per_call)) * cal.headroom; + expect(Number(h.field(second.out, 'BUDGET', 'projected'))).toBe(Math.ceil((3 * K.per_call * factor) / cal.round_to) * cal.round_to); + expect(Number(h.field(second.out, 'BUDGET', 'tail'))).toBe(Math.ceil((2_000_000 * factor) / cal.round_to) * cal.round_to + K.stop_overhead); + expect(second.out).toMatch(/^DRIFT: id=\S+ kind=step next=2 projected=2250000 actual=450000 ratio=0\.20$/m); + const d = h.sb(['check', '--run', s.run, '--step', '3', '--next', 'dispatch:specialist,red-team', '--reading', '900000']); + expect(Number(h.field(d.out, 'BUDGET', 'projected'))).toBe(2 * K.per_dispatch); + }); + + test('an explicit --reserve is kept as given, and the stop overhead keeps its calibrated fit', () => { + const h = fresh(); + const r = h.sb(['start', '--record', h.record(), '--token-cap', '100M', '--reading', '0', '--reserve', '3M']); + const run = h.field(r.out, 'BUDGET_RUN', 'id'); + h.sb(['check', '--run', run, '--step', '1', '--next', '2', '--reading', '450000']); + const c = h.sb(['check', '--run', run, '--step', '2', '--next', '3', '--reading', '900000']); + expect(h.field(c.out, 'BUDGET', 'calibration')).not.toStartWith('step:prior'); + expect(Number(h.field(c.out, 'BUDGET', 'tail'))).toBe(3_000_000 + K.stop_overhead); + }); +}); + +describe('dispatch reservations settle when the reviewer returns (GST-101)', () => { + test('check --returned settles a consumed reservation, so the next per-dispatch admission is not blocked by it', () => { + const h = fresh(); + const s = started(h, '20M'); + const a = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:specialist', '--reading', '0']); + const first = h.field(a.out, 'RESERVATION', 'id'); + expect(h.reviewLog(['--start', 'review', '--reservation', first]).code).toBe(0); + const b = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:specialist', '--reading', '400000', '--returned', first]); + expect(b.code, b.out + b.err).toBe(0); + expect(h.field(b.out, 'BUDGET', 'reserved')).toBe('0'); + expect(h.ledger(s.ledger).reservations.find((r: any) => r.id === first)).toMatchObject({ state: 'settled', actual: 400_000 }); + const bad = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:specialist', '--reading', '400000', '--returned', 'nope']); + expect(bad.code).toBe(2); + expect(bad.err).toContain('--returned'); + }); + + test('--finish settles every consumed dispatch reservation of the pass, and the next reading measures them as one group', () => { + const h = fresh(); + const s = started(h, '80M'); + const tokens: string[] = []; + let reading = 0; + for (const role of ['specialist', 'specialist', 'red-team']) { + const id = h.field(h.sb(['check', '--run', s.run, '--step', '9', '--next', `dispatch:${role}`, '--reading', String(reading)]).out, 'RESERVATION', 'id'); + const st = h.reviewLog(['--start', 'review', '--reservation', id]); + expect(st.code).toBe(0); + tokens.push(st.out.trim()); + reading += 500_000; + } + expect(h.reviewLog([finishRow(), '--finish', tokens[0]!]).code).toBe(0); + const dispatches = () => h.ledger(s.ledger).reservations.filter((r: any) => r.kind === 'dispatch'); + expect(dispatches().map((r: any) => r.state)).toEqual(['settled', 'settled', 'settled']); + const next = h.sb(['check', '--run', s.run, '--step', '9', '--next', '10', '--reading', String(reading)]); + expect(h.field(next.out, 'BUDGET', 'reserved')).toBe('0'); + expect(dispatches().map((r: any) => r.actual)).toEqual([500_000, 500_000, 500_000]); + }); + + test('moving to the next step settles dispatch reservations a reviewer consumed in the step before', () => { + const h = fresh(); + const s = started(h, '80M'); + const id = h.field(h.sb(['check', '--run', s.run, '--step', '11', '--next', 'dispatch:outside-voice', '--reading', '0']).out, 'RESERVATION', 'id'); + expect(h.reviewLog(['--start', 'adversarial-review', '--reservation', id]).code).toBe(0); + const c = h.sb(['check', '--run', s.run, '--step', '11', '--next', '11.5', '--reading', '600000']); + expect(c.code).toBe(0); + expect(h.ledger(s.ledger).reservations.find((r: any) => r.id === id)).toMatchObject({ state: 'settled', actual: 600_000 }); + }); + + test('close prints projected against actual for the run', () => { + const h = fresh(); + const s = started(h, '80M'); + h.sb(['check', '--run', s.run, '--step', '1', '--next', '2', '--reading', '300000']); + const close = h.sb(['close', '--run', s.run, '--reading', '700000']); + expect(close.code).toBe(0); + expect(close.out).toMatch(/^BUDGET_DRIFT: steps=2 step_projected=\d+ step_actual=700000 dispatches=0 dispatch_projected=0 dispatch_actual=0$/m); + }); +}); diff --git a/test/ship-budget.test.ts b/test/ship-budget.test.ts index fcbd635b6..a138a66b1 100644 --- a/test/ship-budget.test.ts +++ b/test/ship-budget.test.ts @@ -11,7 +11,6 @@ import { BUDGET_BIN, budgetHarness, type Harness } from './helpers/ship-budget-h const cal = loadCalibration(); const K = unitConstants(cal, 'capy-tokens'); -const STEP2 = stepProjection(cal, 'capy-tokens', '2'); const harnesses: Harness[] = []; const fresh = () => { const h = budgetHarness(); harnesses.push(h); return h; }; afterEach(() => { while (harnesses.length) harnesses.pop()!.cleanup(); }); @@ -42,12 +41,11 @@ describe('parsing and the admission rule', () => { test('CLI boundary: a check exactly at the cap continues, one token over stops (exit 3)', () => { const h = fresh(); - const cap = 40_000_000; for (const [extra, code, verdict] of [[0, 0, 'continue'], [1, 3, 'stop']] as const) { - const s = started(h, ['--token-cap', String(cap), '--reading', '0', '--reserve', '0']); + const cap = K.per_dispatch + K.stop_overhead - extra; + const s = started(h, ['--token-cap', String(cap), '--reading', '3000000', '--reserve', '0']); expect(s.r.code).toBe(0); - const reading = cap - STEP2 - K.stop_overhead + extra; - const c = h.sb(['check', '--run', s.run, '--step', '1', '--next', '2', '--reading', String(reading)]); + const c = h.sb(['check', '--run', s.run, '--step', '1', '--next', 'dispatch:specialist', '--reading', '3000000']); expect(c.code).toBe(code); expect(h.field(c.out, 'BUDGET', 'verdict')).toBe(verdict); expect(h.field(c.out, 'BUDGET', 'admission_cap')).toBe(String(cap)); @@ -98,7 +96,9 @@ describe('reservations', () => { const spent = 2 * K.per_dispatch; const next = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:red-team', '--reading', String(spent)]); expect(Number(h.field(next.out, 'BUDGET', 'used'))).toBe(spent); - expect(next.code).toBe(spent + K.per_dispatch + K.stop_overhead <= cap ? 0 : 3); + const projected = Number(h.field(next.out, 'BUDGET', 'projected')); + expect(projected).toBe(Math.ceil((K.per_dispatch * 2 * cal.headroom) / cal.round_to) * cal.round_to); + expect(next.code).toBe(spent + projected + K.stop_overhead <= cap ? 0 : 3); const later = h.sb(['check', '--run', s.run, '--step', '9', '--next', 'dispatch:red-team', '--reading', String(cap - K.stop_overhead - K.per_dispatch + 1)]); expect(later.code).toBe(3); }); diff --git a/test/size-ledger/ship/gst-101-ship-budget-calibration.json b/test/size-ledger/ship/gst-101-ship-budget-calibration.json new file mode 100644 index 000000000..b07b26cdb --- /dev/null +++ b/test/size-ledger/ship/gst-101-ship-budget-calibration.json @@ -0,0 +1,4 @@ +{ + "bytes": 146, + "why": "GST-101: budget-handoff names check --returned for reviewers admitted one at a time" +}