Attribute every paid trial's cost: bind label-named records by testName, price aborted SDK sessions from streamed usage, model-free cases cost $0

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

lib/pricing.ts

@@ -143,3 +143,65 @@ export function estimateCostUsd(
   const outputCost = tokens.output * row.output_per_mtok / 1_000_000;
   return +(inputCost + cachedCost + outputCost).toFixed(6);
 }
+
+/** Token classes of one Anthropic Messages usage block, as Claude Code's stream-json and session transcripts carry it. */
+export interface AnthropicUsage {
+  input_tokens?: number;
+  output_tokens?: number;
+  cache_read_input_tokens?: number;
+  cache_creation_input_tokens?: number;
+  cache_creation?: { ephemeral_5m_input_tokens?: number; ephemeral_1h_input_tokens?: number };
+}
+
+/** Anthropic bills a prompt-cache write at 1.25x input for the 5-minute TTL and 2x for the 1-hour TTL. */
+export const CACHE_WRITE_MULTIPLIER = { ephemeral_5m: 1.25, ephemeral_1h: 2 } as const;
+
+/**
+ * Cost of one usage block at its model's row, prompt-cache writes included
+ * (unlike estimateCostUsd). Undefined when the model has no price row: an
+ * unpriced model's usage is unknown cost, never zero.
+ */
+export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | undefined): number | undefined {
+  const row = pricingRow(model);
+  if (!row) return undefined;
+  const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
+  const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens);
+  const fiveMinute = usage.cache_creation ? count(usage.cache_creation.ephemeral_5m_input_tokens) : count(usage.cache_creation_input_tokens);
+  const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1;
+  return (count(usage.input_tokens) * row.input_per_mtok
+    + count(usage.cache_read_input_tokens) * cachedRate
+    + fiveMinute * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_5m
+    + oneHour * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_1h
+    + count(usage.output_tokens) * row.output_per_mtok) / 1_000_000;
+}
+
+/**
+ * Billed cost of a session's assistant messages, subagent (sidechain) turns
+ * included. A message streams as several events sharing one message id whose
+ * usage grows, so each id counts once at its largest output. Each message is
+ * priced at its own model, else `fallbackModel`. Undefined when no message
+ * carries usage or any message's model has no price row.
+ */
+export function streamedUsageCostUsd(
+  events: ReadonlyArray<{ message?: { id?: unknown; model?: unknown; usage?: unknown } } | null | undefined>,
+  fallbackModel?: string,
+): number | undefined {
+  const byId = new Map<string, { model?: string; usage: AnthropicUsage }>();
+  for (const event of events) {
+    const message = event?.message;
+    const usage = message?.usage as AnthropicUsage | undefined;
+    if (!usage || typeof usage !== 'object' || typeof message!.id !== 'string' || message!.model === '<synthetic>') continue;
+    const seen = byId.get(message!.id);
+    if (!seen || (usage.output_tokens ?? 0) >= (seen.usage.output_tokens ?? 0)) {
+      byId.set(message!.id, { model: typeof message!.model === 'string' ? message!.model : fallbackModel, usage });
+    }
+  }
+  if (byId.size === 0) return undefined;
+  let total = 0;
+  for (const { model, usage } of byId.values()) {
+    const cost = anthropicUsageCostUsd(usage, model);
+    if (cost === undefined) return undefined;
+    total += cost;
+  }
+  return +total.toFixed(6);
+}

scripts/harness-version.json

@@ -13,21 +13,21 @@
   "files": {
     "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
-    "scripts/lib/paid-report.ts": {"blob":"224dc5e8ba23a3589a37bad94db72ed1ca3c7c77","since":6,"nonBehavioral":"a case shard's result records are attributed to its one case: the trial's cost_usd and cost_known come from them, and a failed trial's failure_class, exit_reason and error come from its record instead of a bare assertion with the JUnit message, so a red whose record says infra or timeout is classed by that record (INFRA instead of VERDICT: rerun and bisect eligibility, not a pass rate); no case's outcome or selection changes and pass rates count outcomes only, so no pass-rate series resets"},
+    "scripts/lib/paid-report.ts": {"blob":"f1ba68c3e0da4baadabcffb79a3f5656b0b3fd34","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
     "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
     "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
     "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "scripts/test-pr-profile.ts": {"blob":"188710401a08156f585aebcb699151c60e3d6273","since":6,"nonBehavioral":"#3499: RELEASE_METADATA_FILES exempts NOTICE.md, LICENSE and licenses/** and captain/templates/** joins FREE_ONLY_PR_FILES; selection only, no case runs or is judged differently"},
     "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
-    "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
+    "test/helpers/agent-sdk-runner.ts": {"blob":"ecbd0fd8c7864d09ea7c3cabd106558257eeff3f","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
     "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
     "test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
     "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
-    "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/e2e-helpers.ts": {"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
     "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
-    "test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
+    "test/helpers/eval-store.ts": {"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
     "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
     "test/helpers/llm-judge.ts": {"blob":"a3177665b662932ce485ccc976c2b36a1531a0bb","since":6,"nonBehavioral":"callJudge rewrites only the message of a JSON parse failure (names the judge output, stop reason, length and the retained llm-judge-response-parse-error event); it still throws a SyntaxError before any score is parsed, the request is unchanged, and no case's pass or fail changes"},
     "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
@@ -46,8 +46,8 @@
     "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
-    "test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
-    "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/session-ledger.ts": {"blob":"5d8b796ccd422f4e1e2ab395bf6ce5a2e4c0e167","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
+    "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
     "test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"},
     "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},
     "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},

scripts/lib/paid-report.ts

@@ -241,6 +241,20 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
   return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
 }
 
+/** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */
+export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-flow']);
+
+/** A trial's cost: its records' sum; with no record, its sessions' billed or streamed cost; a model-free case with neither, $0. */
+export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
+  const cents = (values: number[]) => Math.round(values.reduce((sum, value) => sum + value, 0) * 100) / 100;
+  if (records.length) return { cost_usd: cents(records.map(r => Number(r.cost_usd) || 0)), ...trialCostKnown(records, sessions) };
+  if (sessions.length) {
+    const costs = sessions.map(s => s.billed !== false ? s.cost_usd : s.streamed_cost_usd);
+    return costs.every(c => typeof c === 'number' && Number.isFinite(c)) ? { cost_usd: cents(costs as number[]) } : { cost_usd: 0, cost_known: false };
+  }
+  return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
+}
+
 /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
 /** The red-census guide the report and every eval:pass-rates view link. */
 export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
@@ -277,11 +291,14 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
         const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped';
         // A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is
         // the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown.
-        const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
-        const failedRecord = mine.find((r: any) => r.passed === false);
+        // A display-label record whose runSkillTest testName is this case (cost_case_id) adds its cost only: the failed record stays the named one.
+        const named = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
+        const mine = [...named, ...records.filter((r: any) => r?.cost_case_id === id && !named.includes(r) && !byCase.has(r?.name))];
+        const failedRecord = named.find((r: any) => r.passed === false);
         const recorded: TrialFailureClass | undefined = !failed ? undefined
           : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
         const sessions = caseSessions(ledger, id, byCase.size === 1);
+        const cost = caseCost(id, mine, sessions);
         const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
         const failureClass = credentials ? 'infra' : recorded;
         const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
@@ -295,7 +312,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
           census.skipped.push(`◌ ${id}  ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
         }
         census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}),
-          costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false,
+          costUsd: cost.cost_usd, costKnown: cost.cost_known !== false,
           ...(failed ? { line: `✗ ${id}  ${kind}  FAIL  ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
             + `${diagnosisText(diagnosis, error)}  [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
             + `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
@@ -303,7 +320,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
         census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
           ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
           ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
-          cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
+          ...cost,
           ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
           quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
       }

test/helpers/agent-sdk-runner.ts

@@ -37,6 +37,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin';
 import { resolveEvalModel } from '../../lib/eval-model';
+import { streamedUsageCostUsd } from '../../lib/pricing';
 import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger';
 import { hermeticChildEnv } from './hermetic-env';
 import type { SkillTestResult } from './session-runner';
@@ -353,11 +354,12 @@ export async function runAgentSdkTest(
       const budget = typeof opts.sessionBudgetMs === 'function' ? opts.sessionBudgetMs() : opts.sessionBudgetMs;
       const aborted = controller.signal.aborted || opts.signal?.aborted === true;
       const structured = observer.verdict();
+      const streamed = observer.billed ? undefined : streamedUsageCostUsd(assistantTurns, model);
       const end: SessionEnd = aborted ? (budget !== undefined && elapsed + 1000 >= budget ? 'session_timeout' : 'aborted') : structured?.end ?? fallback;
       appendSessionLedger({ key: sessionKey('agent-sdk', opts.testName), ...(opts.testName ? { test_name: opts.testName } : {}), runner: 'agent-sdk',
         started_at: startedAt, ...(budget !== undefined ? { budget_ms: budget } : {}), elapsed_ms: elapsed, end,
         ...(end === 'session_timeout' ? { evidence: `armed ${budget}ms session timeout fired` } : structured && !aborted ? { evidence: structured.evidence } : {}),
-        liveness: observer.summary(performance.now()), billed: observer.billed });
+        liveness: observer.summary(performance.now()), billed: observer.billed, ...(streamed !== undefined ? { streamed_cost_usd: streamed } : {}) });
     };
     const controller = new AbortController();
     let activeQuery: ReturnType<QueryProvider> | undefined;

test/helpers/e2e-helpers.ts

@@ -274,6 +274,7 @@ export function recordE2E(
     duration_ms: result.duration,
     cost_usd: result.costEstimate.estimatedCost,
     ...(result.costKnown === false ? { cost_known: false } : {}),
+    ...(result.testName && result.testName !== name && result.testName in E2E_TIERS ? { cost_case_id: result.testName } : {}),
     ...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}),
     transcript: result.transcript,
     output: result.output?.slice(0, 2000),

test/helpers/eval-store.ts

@@ -86,6 +86,9 @@ export interface EvalTestEntry {
   // TRIAL_ENV variables the paid runner sets on an isolated trial shard.
   /** Registry id (E2E_TIERS / LLM_JUDGE_TOUCHFILES key) this record belongs to. */
   case_id?: string;
+  /** Registry id whose cost this record carries (runSkillTest's testName) when its name is a display label;
+   *  the report binds cost by it, never the failed record or its failure class. */
+  cost_case_id?: string;
   kind?: EvalCaseKind;
   /** 1-based trial index within the case's panel. */
   trial?: number;

test/helpers/session-ledger.ts

@@ -52,6 +52,8 @@ export interface SessionLedgerRow {
   liveness?: LivenessSummary;
   /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */
   billed?: boolean;
+  /** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */
+  streamed_cost_usd?: number;
   /** Codex sessions: the model, its billed cost and token classes. */
   model?: string;
   cost_usd?: number;

test/helpers/session-runner.ts

@@ -61,6 +61,8 @@ export interface SkillTestResult {
   tokenUsage?: CodexTokenUsage;
   /** Codex host: false when the session completed no turn or its model has no price row. */
   costKnown?: boolean;
+  /** runSkillTest's testName: the registered case id the session ledger names, which recordE2E binds the record's cost to. */
+  testName?: string;
 }
 
 /** Local default startup grace: 90s covers observed API queue latency
@@ -741,5 +743,5 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af
     turnsUsed,
   };
 
-  return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs };
+  return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs, ...(testName ? { testName } : {}) };
 }

test/paid-report-cost.test.ts

@@ -0,0 +1,103 @@
+/**
+ * PR 2 of GSTA-150: every trial's cost is attributed. A display-label record binds its cost by
+ * runSkillTest's testName (cost_case_id) without changing the failed record or failure class; a case
+ * with no record takes its sessions' streamed cost; a listed model-free case costs a known $0.
+ */
+import { describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { MODEL_FREE_CASES, caseCost, junitCensus } from '../scripts/lib/paid-report';
+import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing';
+import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
+import type { SessionLedgerRow } from './helpers/session-ledger';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const session = (over: Partial<SessionLedgerRow>): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over });
+
+describe('pricing usage', () => {
+  test('cache writes bill 1.25x input (5-minute) and 2x (1-hour); reads at the cached rate', () => {
+    // claude-opus-5-5: input $4, output $20, cache read $0.20 per Mtok.
+    expect(anthropicUsageCostUsd({ input_tokens: 1_000_000, output_tokens: 1_000_000, cache_read_input_tokens: 1_000_000, cache_creation_input_tokens: 1_000_000 }, 'claude-opus-5-5'))
+      .toBeCloseTo(4 + 20 + 0.2 + 5, 6);
+    expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5'))
+      .toBeCloseTo(5 + 16, 6);
+    expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined();
+  });
+
+  test('streamed messages count once per id at their largest output, sidechains included; an unpriced model is unknown', () => {
+    const msg = (id: string, output: number, extra: Record<string, unknown> = {}) => ({ message: { id, model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: output } }, ...extra });
+    expect(streamedUsageCostUsd([msg('a', 10), msg('a', 1_000_000), msg('b', 500_000, { parent_tool_use_id: 'tool-1' })])).toBeCloseTo(30, 6);
+    expect(streamedUsageCostUsd([{ message: { id: 'a', model: '<synthetic>', usage: { output_tokens: 5 } } }])).toBeUndefined();
+    expect(streamedUsageCostUsd([{ message: { id: 'a', model: 'no-such-model', usage: { output_tokens: 5 } } }, msg('b', 5)])).toBeUndefined();
+    expect(streamedUsageCostUsd([{ message: { id: 'a', usage: { output_tokens: 1_000_000 } } }], 'claude-opus-5-5')).toBeCloseTo(20, 6);
+  });
+});
+
+describe('caseCost', () => {
+  test('records decide when present; sessions never add to them', () => {
+    expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 });
+    expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false });
+  });
+
+  test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => {
+    expect(caseCost('auq', [], [session({ billed: false, streamed_cost_usd: 0.123 })])).toEqual({ cost_usd: 0.12 });
+    expect(caseCost('auq', [], [session({ billed: false })])).toEqual({ cost_usd: 0, cost_known: false });
+    expect(caseCost('auq', [], [session({ runner: 'claude-p', billed: true })])).toEqual({ cost_usd: 0, cost_known: false });
+  });
+
+  test('only a listed model-free case with neither record nor session costs a known $0', () => {
+    expect(caseCost('timeline-event-flow', [], [])).toEqual({ cost_usd: 0 });
+    expect(caseCost('some-other-case', [], [])).toEqual({ cost_usd: 0, cost_known: false });
+  });
+
+  test('every listed model-free case body starts no model session', () => {
+    for (const id of MODEL_FREE_CASES) {
+      const file = E2E_TOUCHFILES[id]?.find(entry => /^test\/[^/]+\.test\.ts$/.test(entry));
+      expect(file, id).toBeTruthy();
+      const source = fs.readFileSync(path.join(ROOT, file!), 'utf8');
+      const start = source.search(new RegExp(`IfSelected\\(\\s*['"\`]${id}['"\`]`));
+      expect(start, id).toBeGreaterThan(-1);
+      const rest = source.slice(start + 1);
+      const end = rest.search(/IfSelected\(/);
+      const body = end === -1 ? rest : rest.slice(0, end);
+      expect(body, id).not.toMatch(/runSkillTest|runAgentSdkTest|runCodex|launchClaudePty|runPtySession|claude-pty|callJudge|judge\w*\(/);
+    }
+  });
+});
+
+describe('junitCensus cost binding', () => {
+  const key = 'test/skill-e2e-bws.test.ts';
+  const census = (records: any[], ledger: SessionLedgerRow[] = []) => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cost-'));
+    const shard = path.join(root, 'shards', 'skill-e2e-bws');
+    fs.mkdirSync(shard, { recursive: true });
+    fs.writeFileSync(path.join(shard, 'junit.xml'), `<testsuites><testsuite name="${key}">
+<testcase name="browse-basic" classname="${key}" time="1"></testcase>
+<testcase name="browse-snapshot" classname="${key}" time="1"><failure type="AssertionError" message="expected snapshot"/></testcase>
+</testsuite></testsuites>`);
+    fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), ledger.map(row => JSON.stringify(row)).join('\n'));
+    const manifest: any = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, profile: 'full', selection: { e2e: null, judges: null }, entries: [{ file: key, slice: 1, status: 'planned', estimatedMs: 1 }] };
+    const result: any = { version: 1, tier: 'gate', profile: 'full', selection: { e2e: null, judges: null }, sliceIndex: 1, sliceCount: 1, attempt: 1,
+      outcomes: [{ files: [key], status: 'failed', exitCode: 1, elapsedMs: 10, executedTests: 2, skippedTests: 0 }] };
+    try { return junitCensus(manifest, [{ root, result }], new Map([[key, records]]), {}); } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  };
+  const labelled = [
+    { name: 'browse basic commands', passed: true, cost_usd: 0.05, cost_case_id: 'browse-basic' },
+    { name: 'browse snapshot flags', passed: false, cost_usd: 0.06, cost_case_id: 'browse-snapshot', exit_reason: 'timeout', failure_class: 'infra', error: 'record error' },
+  ];
+
+  test('display-label records bind their cost by cost_case_id in a multi-case shard', () => {
+    const cases = census(labelled).ruleCases;
+    expect(cases.map(c => [c.id, c.costUsd, c.costKnown])).toEqual([['browse-basic', 0.05, true], ['browse-snapshot', 0.06, true]]);
+  });
+
+  test('the failed record and failure class stay what the named records give, as without cost_case_id', () => {
+    const bound = census(labelled).ruleCases.find(c => c.id === 'browse-snapshot')!;
+    const unbound = census(labelled.map(({ cost_case_id, ...rest }) => rest)).ruleCases.find(c => c.id === 'browse-snapshot')!;
+    expect(unbound).toMatchObject({ costUsd: 0, costKnown: false });
+    expect({ failureClass: bound.failureClass, error: bound.error, line: bound.line?.replace(/ {2}\[slice.*$/, '') })
+      .toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') });
+    expect(bound.failureClass).toBe('assertion');
+  });
+});