From 96250a59e074f8bc6cef0ad4775a67be275fc263 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:31:09 +0000 Subject: [PATCH] fix: case cost adds streamed sessions to record sums and keeps record cost_known Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 21 ++++++++++++--------- test/paid-report-cost.test.ts | 6 ++++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index dbf0edd57..5b27d9e0e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"f1ba68c3e0da4baadabcffb79a3f5656b0b3fd34","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, + "scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index f1ba68c3e..b00d662e5 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -9,7 +9,7 @@ import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets'; import { isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, - trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail, + trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; import { publishedFence, sanitizeFixedFenceLine } from './published-text'; @@ -244,15 +244,18 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): /** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-flow']); -/** A trial's cost: its records' sum; with no record, its sessions' billed or streamed cost; a model-free case with neither, $0. */ +/** + * A trial's cost: its records' sum plus what no record carries, a session without a terminal result priced from its + * streamed usage (streamed_cost_usd). With no record, a billed session counts its own cost_usd (Codex). Unknown when any + * record or session cost is; a listed model-free case with neither record nor session is a known $0. + */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { - const cents = (values: number[]) => Math.round(values.reduce((sum, value) => sum + value, 0) * 100) / 100; - if (records.length) return { cost_usd: cents(records.map(r => Number(r.cost_usd) || 0)), ...trialCostKnown(records, sessions) }; - if (sessions.length) { - const costs = sessions.map(s => s.billed !== false ? s.cost_usd : s.streamed_cost_usd); - return costs.every(c => typeof c === 'number' && Number.isFinite(c)) ? { cost_usd: cents(costs as number[]) } : { cost_usd: 0, cost_known: false }; - } - return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; + if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; + const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); + const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); + const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))] + .reduce((sum, c) => sum + c, 0); + return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 98340c9fd..7ceb8af1b 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -35,9 +35,11 @@ describe('pricing usage', () => { }); describe('caseCost', () => { - test('records decide when present; sessions never add to them', () => { + test('a billed session\'s cost is already in the records; a streamed session adds what no record carries', () => { expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 }); - expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false }); + expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0.4 }); + expect(caseCost('x', [{ cost_usd: 0.2 }], [session({ runner: 'pty', billed: false })])).toEqual({ cost_usd: 0.2, cost_known: false }); + expect(caseCost('x', [{ cost_usd: 0.2, cost_known: false }], [])).toEqual({ cost_usd: 0.2, cost_known: false }); }); test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => {