From 8b572eefecac11c046111de818e2c9ac2bc2ed35 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:50:30 +0000 Subject: [PATCH] fix: the headline adds only streamed session cost; cache writes without a 5m key Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/pricing.ts | 3 ++- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 11 +++-------- test/paid-report-census.test.ts | 9 +++++++++ test/paid-report-cost.test.ts | 2 ++ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/pricing.ts b/lib/pricing.ts index 0e59f3690..6b7697790 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -166,7 +166,8 @@ export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | und if (!row) return undefined; const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0; const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens); - const fiveMinute = usage.cache_creation ? count(usage.cache_creation.ephemeral_5m_input_tokens) : count(usage.cache_creation_input_tokens); + const fiveMinute = typeof usage.cache_creation?.ephemeral_5m_input_tokens === 'number' + ? count(usage.cache_creation.ephemeral_5m_input_tokens) : Math.max(0, count(usage.cache_creation_input_tokens) - oneHour); const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1; return (count(usage.input_tokens) * row.input_per_mtok + count(usage.cache_read_input_tokens) * cachedRate diff --git a/scripts/harness-version.json b/scripts/harness-version.json index d2993d593..e225bbbad 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"a4ef799c233c65e9209a4b654360476fdbf51802","since":6,"nonBehavioral":"the trial model comes from the named records only, as before cost_case_id; cost fields only, no verdict, failure class or series key changes"}, + "scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index a4ef799c2..e0d18da4e 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -230,7 +230,7 @@ export interface JUnitCensus { deselected: Map; /** Per shard key: skip reason -> skipped testcase count. */ shardSkipReasons: Map>; - /** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */ + /** Streamed session cost (sessions without a terminal result), which no collector record carries; the headline adds it to the collectors' total_cost_usd. */ ledgerCostUsd: number; /** ` :: ()` for every testcase no rule attributes. */ unattributed: string[]; @@ -253,17 +253,12 @@ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-fl */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; - const unrecorded = unrecordedCosts(records, sessions); + const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded); return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } -/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */ -function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array { - return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); -} - const knownSum = (costs: Array) => costs.reduce((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0); /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ @@ -311,7 +306,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const sessions = caseSessions(ledger, id, byCase.size === 1); const cost = caseCost(id, mine, sessions); - census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions)); + census.ledgerCostUsd += knownSum(sessions.map(s => s.billed === false ? s.streamed_cost_usd : 0)); const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; const failureClass = credentials ? 'infra' : recorded; const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 15f483f99..103e1c8d2 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -137,6 +137,15 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown'); }); + test('a billed session\'s cost is already in its collector total, so the verdict never adds it again (Codex host, unbound label record)', () => { + const r = report('periodic', dir => { + results(dir, 'skill-e2e-cso', [{ name: '/cso full audit', suite: 'cso', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.5 }]); + fs.writeFileSync(path.join(shard(dir, 'skill-e2e-cso'), 'session-ledger.jsonl'), + JSON.stringify({ key: 'k', runner: 'codex', started_at: 't', elapsed_ms: 1, end: 'completed', billed: true, cost_usd: 0.5 }) + '\n'); + }); + expect(collector(r.dir).verdict.costUsd).toBe(0.5); + }); + // Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard // wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class); // fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped; diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 174d8be38..ac5e08eff 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -23,6 +23,8 @@ describe('pricing usage', () => { .toBeCloseTo(4 + 20 + 0.2 + 5, 6); expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5')) .toBeCloseTo(5 + 16, 6); + expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5')) + .toBeCloseTo(5 + 16, 6); expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined(); });