fix: case cost adds streamed sessions to record sums and keeps record cost_known

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-29 04:31:09 +00:00
1 parent d59225dd74
commit 96250a59e0
3 files changed
+17 -12

No files matched your search

+1 -1
View File
@@ -13,7 +13,7 @@
"files": {
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
"scripts/lib/paid-report.ts": {"blob":"f1ba68c3e0da4baadabcffb79a3f5656b0b3fd34","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
"scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
+12 -9
View File
@@ -9,7 +9,7 @@ import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
import {
isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
trialFailureFields, trialSessions, type TrialFailureDetail,
} from '../../test/helpers/eval-store';
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
@@ -244,15 +244,18 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
/** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */
export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-flow']);
/** A trial's cost: its records' sum; with no record, its sessions' billed or streamed cost; a model-free case with neither, $0. */
/**
* A trial's cost: its records' sum plus what no record carries, a session without a terminal result priced from its
* streamed usage (streamed_cost_usd). With no record, a billed session counts its own cost_usd (Codex). Unknown when any
* record or session cost is; a listed model-free case with neither record nor session is a known $0.
*/
export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
const cents = (values: number[]) => Math.round(values.reduce((sum, value) => sum + value, 0) * 100) / 100;
if (records.length) return { cost_usd: cents(records.map(r => Number(r.cost_usd) || 0)), ...trialCostKnown(records, sessions) };
if (sessions.length) {
const costs = sessions.map(s => s.billed !== false ? s.cost_usd : s.streamed_cost_usd);
return costs.every(c => typeof c === 'number' && Number.isFinite(c)) ? { cost_usd: cents(costs as number[]) } : { cost_usd: 0, cost_known: false };
}
return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c));
const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))]
.reduce((sum, c) => sum + c, 0);
return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) };
}
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */