From d59225dd7459ed3b695be5553d828443c808e587 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:27:50 +0000 Subject: [PATCH 01/35] Attribute every paid trial's cost: bind label-named records by testName, price aborted SDK sessions from streamed usage, model-free cases cost $0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/pricing.ts | 62 +++++++++++++++++++ scripts/harness-version.json | 12 ++-- scripts/lib/paid-report.ts | 25 ++++++-- test/helpers/agent-sdk-runner.ts | 4 +- test/helpers/e2e-helpers.ts | 1 + test/helpers/eval-store.ts | 3 + test/helpers/session-ledger.ts | 2 + test/helpers/session-runner.ts | 4 +- test/paid-report-cost.test.ts | 103 +++++++++++++++++++++++++++++++ 9 files changed, 204 insertions(+), 12 deletions(-) create mode 100644 test/paid-report-cost.test.ts diff --git a/lib/pricing.ts b/lib/pricing.ts index 3fd3a3421..0e59f3690 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -143,3 +143,65 @@ export function estimateCostUsd( const outputCost = tokens.output * row.output_per_mtok / 1_000_000; return +(inputCost + cachedCost + outputCost).toFixed(6); } + +/** Token classes of one Anthropic Messages usage block, as Claude Code's stream-json and session transcripts carry it. */ +export interface AnthropicUsage { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; + cache_creation?: { ephemeral_5m_input_tokens?: number; ephemeral_1h_input_tokens?: number }; +} + +/** Anthropic bills a prompt-cache write at 1.25x input for the 5-minute TTL and 2x for the 1-hour TTL. */ +export const CACHE_WRITE_MULTIPLIER = { ephemeral_5m: 1.25, ephemeral_1h: 2 } as const; + +/** + * Cost of one usage block at its model's row, prompt-cache writes included + * (unlike estimateCostUsd). Undefined when the model has no price row: an + * unpriced model's usage is unknown cost, never zero. + */ +export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | undefined): number | undefined { + const row = pricingRow(model); + if (!row) return undefined; + const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0; + const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens); + const fiveMinute = usage.cache_creation ? count(usage.cache_creation.ephemeral_5m_input_tokens) : count(usage.cache_creation_input_tokens); + const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1; + return (count(usage.input_tokens) * row.input_per_mtok + + count(usage.cache_read_input_tokens) * cachedRate + + fiveMinute * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_5m + + oneHour * row.input_per_mtok * CACHE_WRITE_MULTIPLIER.ephemeral_1h + + count(usage.output_tokens) * row.output_per_mtok) / 1_000_000; +} + +/** + * Billed cost of a session's assistant messages, subagent (sidechain) turns + * included. A message streams as several events sharing one message id whose + * usage grows, so each id counts once at its largest output. Each message is + * priced at its own model, else `fallbackModel`. Undefined when no message + * carries usage or any message's model has no price row. + */ +export function streamedUsageCostUsd( + events: ReadonlyArray<{ message?: { id?: unknown; model?: unknown; usage?: unknown } } | null | undefined>, + fallbackModel?: string, +): number | undefined { + const byId = new Map(); + for (const event of events) { + const message = event?.message; + const usage = message?.usage as AnthropicUsage | undefined; + if (!usage || typeof usage !== 'object' || typeof message!.id !== 'string' || message!.model === '') continue; + const seen = byId.get(message!.id); + if (!seen || (usage.output_tokens ?? 0) >= (seen.usage.output_tokens ?? 0)) { + byId.set(message!.id, { model: typeof message!.model === 'string' ? message!.model : fallbackModel, usage }); + } + } + if (byId.size === 0) return undefined; + let total = 0; + for (const { model, usage } of byId.values()) { + const cost = anthropicUsageCostUsd(usage, model); + if (cost === undefined) return undefined; + total += cost; + } + return +total.toFixed(6); +} diff --git a/scripts/harness-version.json b/scripts/harness-version.json index f71515581..dbf0edd57 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,21 +13,21 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"224dc5e8ba23a3589a37bad94db72ed1ca3c7c77","since":6,"nonBehavioral":"a case shard's result records are attributed to its one case: the trial's cost_usd and cost_known come from them, and a failed trial's failure_class, exit_reason and error come from its record instead of a bare assertion with the JUnit message, so a red whose record says infra or timeout is classed by that record (INFRA instead of VERDICT: rerun and bisect eligibility, not a pass rate); no case's outcome or selection changes and pass rates count outcomes only, so no pass-rate series resets"}, + "scripts/lib/paid-report.ts": {"blob":"f1ba68c3e0da4baadabcffb79a3f5656b0b3fd34","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "scripts/test-pr-profile.ts": {"blob":"188710401a08156f585aebcb699151c60e3d6273","since":6,"nonBehavioral":"#3499: RELEASE_METADATA_FILES exempts NOTICE.md, LICENSE and licenses/** and captain/templates/** joins FREE_ONLY_PR_FILES; selection only, no case runs or is judged differently"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, - "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, + "test/helpers/agent-sdk-runner.ts": {"blob":"ecbd0fd8c7864d09ea7c3cabd106558257eeff3f","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, "test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/e2e-helpers.ts": {"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/eval-store.ts": {"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"a3177665b662932ce485ccc976c2b36a1531a0bb","since":6,"nonBehavioral":"callJudge rewrites only the message of a JSON parse failure (names the judge output, stop reason, length and the retained llm-judge-response-parse-error event); it still throws a SyntaxError before any score is parsed, the request is unchanged, and no case's pass or fail changes"}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -46,8 +46,8 @@ "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-ledger.ts": {"blob":"5d8b796ccd422f4e1e2ab395bf6ce5a2e4c0e167","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, + "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"}, "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 224dc5e8b..f1ba68c3e 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -241,6 +241,20 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id]))); } +/** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */ +export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-flow']); + +/** A trial's cost: its records' sum; with no record, its sessions' billed or streamed cost; a model-free case with neither, $0. */ +export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { + const cents = (values: number[]) => Math.round(values.reduce((sum, value) => sum + value, 0) * 100) / 100; + if (records.length) return { cost_usd: cents(records.map(r => Number(r.cost_usd) || 0)), ...trialCostKnown(records, sessions) }; + if (sessions.length) { + const costs = sessions.map(s => s.billed !== false ? s.cost_usd : s.streamed_cost_usd); + return costs.every(c => typeof c === 'number' && Number.isFinite(c)) ? { cost_usd: cents(costs as number[]) } : { cost_usd: 0, cost_known: false }; + } + return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; +} + /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ /** The red-census guide the report and every eval:pass-rates view link. */ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; @@ -277,11 +291,14 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped'; // A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is // the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown. - const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id)); - const failedRecord = mine.find((r: any) => r.passed === false); + // A display-label record whose runSkillTest testName is this case (cost_case_id) adds its cost only: the failed record stays the named one. + const named = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id)); + const mine = [...named, ...records.filter((r: any) => r?.cost_case_id === id && !named.includes(r) && !byCase.has(r?.name))]; + const failedRecord = named.find((r: any) => r.passed === false); const recorded: TrialFailureClass | undefined = !failed ? undefined : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const sessions = caseSessions(ledger, id, byCase.size === 1); + const cost = caseCost(id, mine, sessions); const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; const failureClass = credentials ? 'infra' : recorded; const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; @@ -295,7 +312,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`); } census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}), - costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false, + costUsd: cost.cost_usd, costKnown: cost.cost_known !== false, ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}` + `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]` + `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}` @@ -303,7 +320,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs), - cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions), + ...cost, ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); } diff --git a/test/helpers/agent-sdk-runner.ts b/test/helpers/agent-sdk-runner.ts index 210bba92d..ecbd0fd8c 100644 --- a/test/helpers/agent-sdk-runner.ts +++ b/test/helpers/agent-sdk-runner.ts @@ -37,6 +37,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin'; import { resolveEvalModel } from '../../lib/eval-model'; +import { streamedUsageCostUsd } from '../../lib/pricing'; import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; import { hermeticChildEnv } from './hermetic-env'; import type { SkillTestResult } from './session-runner'; @@ -353,11 +354,12 @@ export async function runAgentSdkTest( const budget = typeof opts.sessionBudgetMs === 'function' ? opts.sessionBudgetMs() : opts.sessionBudgetMs; const aborted = controller.signal.aborted || opts.signal?.aborted === true; const structured = observer.verdict(); + const streamed = observer.billed ? undefined : streamedUsageCostUsd(assistantTurns, model); const end: SessionEnd = aborted ? (budget !== undefined && elapsed + 1000 >= budget ? 'session_timeout' : 'aborted') : structured?.end ?? fallback; appendSessionLedger({ key: sessionKey('agent-sdk', opts.testName), ...(opts.testName ? { test_name: opts.testName } : {}), runner: 'agent-sdk', started_at: startedAt, ...(budget !== undefined ? { budget_ms: budget } : {}), elapsed_ms: elapsed, end, ...(end === 'session_timeout' ? { evidence: `armed ${budget}ms session timeout fired` } : structured && !aborted ? { evidence: structured.evidence } : {}), - liveness: observer.summary(performance.now()), billed: observer.billed }); + liveness: observer.summary(performance.now()), billed: observer.billed, ...(streamed !== undefined ? { streamed_cost_usd: streamed } : {}) }); }; const controller = new AbortController(); let activeQuery: ReturnType | undefined; diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 6ee6937e6..91996ff2b 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -274,6 +274,7 @@ export function recordE2E( duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, ...(result.costKnown === false ? { cost_known: false } : {}), + ...(result.testName && result.testName !== name && result.testName in E2E_TIERS ? { cost_case_id: result.testName } : {}), ...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}), transcript: result.transcript, output: result.output?.slice(0, 2000), diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index a50ea2c28..4085ad18c 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -86,6 +86,9 @@ export interface EvalTestEntry { // TRIAL_ENV variables the paid runner sets on an isolated trial shard. /** Registry id (E2E_TIERS / LLM_JUDGE_TOUCHFILES key) this record belongs to. */ case_id?: string; + /** Registry id whose cost this record carries (runSkillTest's testName) when its name is a display label; + * the report binds cost by it, never the failed record or its failure class. */ + cost_case_id?: string; kind?: EvalCaseKind; /** 1-based trial index within the case's panel. */ trial?: number; diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index a10bf28bf..5d8b796cc 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -52,6 +52,8 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; + /** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */ + streamed_cost_usd?: number; /** Codex sessions: the model, its billed cost and token classes. */ model?: string; cost_usd?: number; diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index cc65a66ad..c73b2596d 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -61,6 +61,8 @@ export interface SkillTestResult { tokenUsage?: CodexTokenUsage; /** Codex host: false when the session completed no turn or its model has no price row. */ costKnown?: boolean; + /** runSkillTest's testName: the registered case id the session ledger names, which recordE2E binds the record's cost to. */ + testName?: string; } /** Local default startup grace: 90s covers observed API queue latency @@ -741,5 +743,5 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af turnsUsed, }; - return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs }; + return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs, ...(testName ? { testName } : {}) }; } diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts new file mode 100644 index 000000000..98340c9fd --- /dev/null +++ b/test/paid-report-cost.test.ts @@ -0,0 +1,103 @@ +/** + * PR 2 of GSTA-150: every trial's cost is attributed. A display-label record binds its cost by + * runSkillTest's testName (cost_case_id) without changing the failed record or failure class; a case + * with no record takes its sessions' streamed cost; a listed model-free case costs a known $0. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { MODEL_FREE_CASES, caseCost, junitCensus } from '../scripts/lib/paid-report'; +import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing'; +import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; +import type { SessionLedgerRow } from './helpers/session-ledger'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const session = (over: Partial): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over }); + +describe('pricing usage', () => { + test('cache writes bill 1.25x input (5-minute) and 2x (1-hour); reads at the cached rate', () => { + // claude-opus-5-5: input $4, output $20, cache read $0.20 per Mtok. + expect(anthropicUsageCostUsd({ input_tokens: 1_000_000, output_tokens: 1_000_000, cache_read_input_tokens: 1_000_000, cache_creation_input_tokens: 1_000_000 }, 'claude-opus-5-5')) + .toBeCloseTo(4 + 20 + 0.2 + 5, 6); + expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5')) + .toBeCloseTo(5 + 16, 6); + expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined(); + }); + + test('streamed messages count once per id at their largest output, sidechains included; an unpriced model is unknown', () => { + const msg = (id: string, output: number, extra: Record = {}) => ({ message: { id, model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: output } }, ...extra }); + expect(streamedUsageCostUsd([msg('a', 10), msg('a', 1_000_000), msg('b', 500_000, { parent_tool_use_id: 'tool-1' })])).toBeCloseTo(30, 6); + expect(streamedUsageCostUsd([{ message: { id: 'a', model: '', usage: { output_tokens: 5 } } }])).toBeUndefined(); + expect(streamedUsageCostUsd([{ message: { id: 'a', model: 'no-such-model', usage: { output_tokens: 5 } } }, msg('b', 5)])).toBeUndefined(); + expect(streamedUsageCostUsd([{ message: { id: 'a', usage: { output_tokens: 1_000_000 } } }], 'claude-opus-5-5')).toBeCloseTo(20, 6); + }); +}); + +describe('caseCost', () => { + test('records decide when present; sessions never add to them', () => { + expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 }); + expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false }); + }); + + test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => { + expect(caseCost('auq', [], [session({ billed: false, streamed_cost_usd: 0.123 })])).toEqual({ cost_usd: 0.12 }); + expect(caseCost('auq', [], [session({ billed: false })])).toEqual({ cost_usd: 0, cost_known: false }); + expect(caseCost('auq', [], [session({ runner: 'claude-p', billed: true })])).toEqual({ cost_usd: 0, cost_known: false }); + }); + + test('only a listed model-free case with neither record nor session costs a known $0', () => { + expect(caseCost('timeline-event-flow', [], [])).toEqual({ cost_usd: 0 }); + expect(caseCost('some-other-case', [], [])).toEqual({ cost_usd: 0, cost_known: false }); + }); + + test('every listed model-free case body starts no model session', () => { + for (const id of MODEL_FREE_CASES) { + const file = E2E_TOUCHFILES[id]?.find(entry => /^test\/[^/]+\.test\.ts$/.test(entry)); + expect(file, id).toBeTruthy(); + const source = fs.readFileSync(path.join(ROOT, file!), 'utf8'); + const start = source.search(new RegExp(`IfSelected\\(\\s*['"\`]${id}['"\`]`)); + expect(start, id).toBeGreaterThan(-1); + const rest = source.slice(start + 1); + const end = rest.search(/IfSelected\(/); + const body = end === -1 ? rest : rest.slice(0, end); + expect(body, id).not.toMatch(/runSkillTest|runAgentSdkTest|runCodex|launchClaudePty|runPtySession|claude-pty|callJudge|judge\w*\(/); + } + }); +}); + +describe('junitCensus cost binding', () => { + const key = 'test/skill-e2e-bws.test.ts'; + const census = (records: any[], ledger: SessionLedgerRow[] = []) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cost-')); + const shard = path.join(root, 'shards', 'skill-e2e-bws'); + fs.mkdirSync(shard, { recursive: true }); + fs.writeFileSync(path.join(shard, 'junit.xml'), ` + + +`); + fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), ledger.map(row => JSON.stringify(row)).join('\n')); + const manifest: any = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, profile: 'full', selection: { e2e: null, judges: null }, entries: [{ file: key, slice: 1, status: 'planned', estimatedMs: 1 }] }; + const result: any = { version: 1, tier: 'gate', profile: 'full', selection: { e2e: null, judges: null }, sliceIndex: 1, sliceCount: 1, attempt: 1, + outcomes: [{ files: [key], status: 'failed', exitCode: 1, elapsedMs: 10, executedTests: 2, skippedTests: 0 }] }; + try { return junitCensus(manifest, [{ root, result }], new Map([[key, records]]), {}); } finally { fs.rmSync(root, { recursive: true, force: true }); } + }; + const labelled = [ + { name: 'browse basic commands', passed: true, cost_usd: 0.05, cost_case_id: 'browse-basic' }, + { name: 'browse snapshot flags', passed: false, cost_usd: 0.06, cost_case_id: 'browse-snapshot', exit_reason: 'timeout', failure_class: 'infra', error: 'record error' }, + ]; + + test('display-label records bind their cost by cost_case_id in a multi-case shard', () => { + const cases = census(labelled).ruleCases; + expect(cases.map(c => [c.id, c.costUsd, c.costKnown])).toEqual([['browse-basic', 0.05, true], ['browse-snapshot', 0.06, true]]); + }); + + test('the failed record and failure class stay what the named records give, as without cost_case_id', () => { + const bound = census(labelled).ruleCases.find(c => c.id === 'browse-snapshot')!; + const unbound = census(labelled.map(({ cost_case_id, ...rest }) => rest)).ruleCases.find(c => c.id === 'browse-snapshot')!; + expect(unbound).toMatchObject({ costUsd: 0, costKnown: false }); + expect({ failureClass: bound.failureClass, error: bound.error, line: bound.line?.replace(/ {2}\[slice.*$/, '') }) + .toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') }); + expect(bound.failureClass).toBe('assertion'); + }); +}); From 96250a59e074f8bc6cef0ad4775a67be275fc263 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:31:09 +0000 Subject: [PATCH 02/35] fix: case cost adds streamed sessions to record sums and keeps record cost_known Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 21 ++++++++++++--------- test/paid-report-cost.test.ts | 6 ++++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index dbf0edd57..5b27d9e0e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"f1ba68c3e0da4baadabcffb79a3f5656b0b3fd34","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, + "scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index f1ba68c3e..b00d662e5 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -9,7 +9,7 @@ import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets'; import { isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, - trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail, + trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; import { publishedFence, sanitizeFixedFenceLine } from './published-text'; @@ -244,15 +244,18 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): /** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-flow']); -/** A trial's cost: its records' sum; with no record, its sessions' billed or streamed cost; a model-free case with neither, $0. */ +/** + * A trial's cost: its records' sum plus what no record carries, a session without a terminal result priced from its + * streamed usage (streamed_cost_usd). With no record, a billed session counts its own cost_usd (Codex). Unknown when any + * record or session cost is; a listed model-free case with neither record nor session is a known $0. + */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { - const cents = (values: number[]) => Math.round(values.reduce((sum, value) => sum + value, 0) * 100) / 100; - if (records.length) return { cost_usd: cents(records.map(r => Number(r.cost_usd) || 0)), ...trialCostKnown(records, sessions) }; - if (sessions.length) { - const costs = sessions.map(s => s.billed !== false ? s.cost_usd : s.streamed_cost_usd); - return costs.every(c => typeof c === 'number' && Number.isFinite(c)) ? { cost_usd: cents(costs as number[]) } : { cost_usd: 0, cost_known: false }; - } - return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; + if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; + const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); + const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); + const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))] + .reduce((sum, c) => sum + c, 0); + return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 98340c9fd..7ceb8af1b 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -35,9 +35,11 @@ describe('pricing usage', () => { }); describe('caseCost', () => { - test('records decide when present; sessions never add to them', () => { + test('a billed session\'s cost is already in the records; a streamed session adds what no record carries', () => { expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 }); - expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false }); + expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0.4 }); + expect(caseCost('x', [{ cost_usd: 0.2 }], [session({ runner: 'pty', billed: false })])).toEqual({ cost_usd: 0.2, cost_known: false }); + expect(caseCost('x', [{ cost_usd: 0.2, cost_known: false }], [])).toEqual({ cost_usd: 0.2, cost_known: false }); }); test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => { From fa5629423dae13932d192ca2740e452f01a05df2 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:43:14 +0000 Subject: [PATCH 03/35] fix: the paid report headline counts ledger-only session cost once Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 20 +++++++++++++++----- test/paid-report-census.test.ts | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 5b27d9e0e..22c6c416b 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"}, + "scripts/lib/paid-report.ts": {"blob":"6adbde198309328bae17c3b17d199963e19a4c7e","since":6,"nonBehavioral":"the paid report headline adds session cost no collector record carries; cost fields only, no verdict or failure class changes"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index b00d662e5..6adbde198 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -230,6 +230,8 @@ export interface JUnitCensus { deselected: Map; /** Per shard key: skip reason -> skipped testcase count. */ shardSkipReasons: Map>; + /** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */ + ledgerCostUsd: number; /** ` :: ()` for every testcase no rule attributes. */ unattributed: string[]; } @@ -251,13 +253,19 @@ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-fl */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; - const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); + const unrecorded = unrecordedCosts(records, sessions); const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); - const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))] - .reduce((sum, c) => sum + c, 0); + const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded); return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } +/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */ +function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array { + return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); +} + +const knownSum = (costs: Array) => costs.reduce((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0); + /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ /** The red-census guide the report and every eval:pass-rates view link. */ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; @@ -269,7 +277,8 @@ export function sliceArtifactName(root: string, slice: number, attempt: number): export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, base: Record, cliVersion?: string): JUnitCensus { - const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; + const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [], + ledgerCostUsd: 0 }; const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry])); for (const { root, result } of artifacts) { for (const outcome of result.outcomes) { @@ -302,6 +311,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const sessions = caseSessions(ledger, id, byCase.size === 1); const cost = caseCost(id, mine, sessions); + census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions)); const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; const failureClass = credentials ? 'infra' : recorded; const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; @@ -796,7 +806,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...(spend.executions ? { reruns: spend } : {}), actionRequired: verdict.problems.length, wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, - costUsd: Math.round(costUsd * 100) / 100, + costUsd: Math.round((costUsd + census.ledgerCostUsd) * 100) / 100, redispatchEligible: red && infraOnly(verdict.problems), }; const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped'); diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index ff9dbf168..15f483f99 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -123,6 +123,20 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr expect(collector(r.dir).verdict.costUsd).toBe(0.75); }); + test('a record-less trial\'s streamed session cost reaches the trial and the verdict total once (GSTA-150 outside voice: $0.65 known, headline $0.00)', () => { + const ledger = (dir: string, slug: string, costs: number[]) => fs.writeFileSync(path.join(shard(dir, slug), 'session-ledger.jsonl'), + costs.map(cost => JSON.stringify({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', billed: false, streamed_cost_usd: cost })).join('\n') + '\n'); + const r = report('periodic', dir => { + ledger(dir, 'skill-e2e-qa-workflow--qa-fix-loop', [0.4, 0.25]); + results(dir, 'skill-e2e-benchmark-providers', [{ name: 'benchmark-providers-live', suite: 's', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.1 }]); + ledger(dir, 'skill-e2e-benchmark-providers', [0.5]); + }); + expect(r.history.find(record => record.case === 'qa-fix-loop')).toMatchObject({ outcome: 'passed', cost_usd: 0.65 }); + expect(r.history.find(record => record.case === 'benchmark-providers-live')).toMatchObject({ cost_usd: 0.6 }); + expect(collector(r.dir).verdict.costUsd).toBe(1.25); + expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown'); + }); + // Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard // wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class); // fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped; From b889070fb965807004fb40f018d312bb3b9cf4b7 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:45:05 +0000 Subject: [PATCH 04/35] fix: a cost-only record never sets the trial model Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 2 +- test/paid-report-cost.test.ts | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 22c6c416b..d2993d593 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"6adbde198309328bae17c3b17d199963e19a4c7e","since":6,"nonBehavioral":"the paid report headline adds session cost no collector record carries; cost fields only, no verdict or failure class changes"}, + "scripts/lib/paid-report.ts": {"blob":"a4ef799c233c65e9209a4b654360476fdbf51802","since":6,"nonBehavioral":"the trial model comes from the named records only, as before cost_case_id; cost fields only, no verdict, failure class or series key changes"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 6adbde198..a4ef799c2 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -334,7 +334,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs), ...cost, - ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), + ...(typeof named[0]?.model === 'string' ? { model: named[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); } census.shardSkipReasons.set(key, reasons); diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 7ceb8af1b..073949b7c 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -102,4 +102,9 @@ describe('junitCensus cost binding', () => { .toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') }); expect(bound.failureClass).toBe('assertion'); }); + + test('a cost-only record never sets the trial\'s model, which keys its pass-rate series', () => { + const history = census(labelled.map(r => ({ ...r, model: 'claude-opus-5-5' }))).history; + expect(history.map(h => [h.case, h.model])).toEqual([['browse-basic', undefined], ['browse-snapshot', undefined]]); + }); }); From 4aec7804e0f8dd7fc8badd8f7e8a7e4671f54d29 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:45:28 +0000 Subject: [PATCH 05/35] test: recordE2E writes cost_case_id only for a registered label-named case Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/paid-report-cost.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 073949b7c..174d8be38 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -11,6 +11,7 @@ import { MODEL_FREE_CASES, caseCost, junitCensus } from '../scripts/lib/paid-rep import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing'; import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; import type { SessionLedgerRow } from './helpers/session-ledger'; +import { recordE2E } from './helpers/e2e-helpers'; const ROOT = path.resolve(import.meta.dir, '..'); const session = (over: Partial): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over }); @@ -68,6 +69,23 @@ describe('caseCost', () => { }); }); +describe('recordE2E cost_case_id', () => { + const entry = (name: string, testName?: string) => { + const tests: any[] = []; + const result: any = { toolCalls: [], browseErrors: [], exitReason: 'success', duration: 1, output: '', transcript: [], model: 'claude-opus-5-5', + costEstimate: { estimatedCost: 0.05, turnsUsed: 1, estimatedTokens: 1 }, ...(testName ? { testName } : {}) }; + recordE2E({ addTest: (t: any) => tests.push(t) } as any, name, 'suite', result); + return tests[0]; + }; + + test('a display-label record carries its registered runSkillTest testName as cost_case_id, and nothing else does', () => { + expect(entry('browse basic commands', 'browse-basic')).toMatchObject({ name: 'browse basic commands', cost_case_id: 'browse-basic' }); + expect(entry('browse-basic', 'browse-basic').cost_case_id).toBeUndefined(); + expect(entry('browse basic commands', 'not-a-registered-case').cost_case_id).toBeUndefined(); + expect(entry('browse basic commands').cost_case_id).toBeUndefined(); + }); +}); + describe('junitCensus cost binding', () => { const key = 'test/skill-e2e-bws.test.ts'; const census = (records: any[], ledger: SessionLedgerRow[] = []) => { From 8364871096a1c33e095c34ff718a328f8e334c62 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:49:04 +0000 Subject: [PATCH 06/35] docs: CHANGELOG for paid-report cost attribution (Unreleased) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6cfab56f..31ded5130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [Unreleased] + +**The train gate's paid report now prices the trials it used to call "cost unknown".** 32 of 139 gate trials had no cost on every gate since Oct 26, so the report's total understated a gate by about $7.50 (GSTA-150 audit). Three of the four causes are fixed here; PTY sessions follow in the next PR. + +### Fixed +- **Label-named records in multi-case shards** (`test/helpers/e2e-helpers.ts`, `scripts/lib/paid-report.ts`): a `claude -p` record named by its display label ("browse basic commands") carries `cost_case_id` from `runSkillTest`'s registered test name, and the report binds its cost to that case. The failed record, failure class and model still come from the records named by case id, so no verdict, class or pass-rate series changes (22 trials: bws, session-intelligence, workflow, skillify, review-attribution, review, gate-integrity, diagram). +- **Aborted Agent SDK sessions** (`test/helpers/agent-sdk-runner.ts`, `lib/pricing.ts`): a session that ends without a result event logs its streamed usage as `streamed_cost_usd` in the session ledger, priced once per message id at its largest output, subagent turns included, with 5-minute cache writes at 1.25x and 1-hour writes at 2x input. An unpriced model leaves the cost unknown (auq-format-gate). +- **Model-free cases**: `timeline-event-flow` runs no model, so with no record and no session it is a known $0. A test checks every listed case's body starts no model session. +- **The headline total** adds session cost that no collector record carries, so newly priced trials reach the gate's `cost $X` line exactly once. + +### Tests +- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. + ## [1.91.304.0] - 2026-10-29 **Running the Capy installer for a second workflow on the same machine no longer takes away the first one's admission.** /ship's docs step runs `scripts/install.sh --for document-release`, and that run replaced the `--for ship` install verdict with its narrower one, so the rest of /ship refused with "the install verdict certified for=document-release, which does not cover codex that ship needs" until someone reran `--for ship` (#3561, seen on gstack-evals#13). From 8b572eefecac11c046111de818e2c9ac2bc2ed35 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:50:30 +0000 Subject: [PATCH 07/35] fix: the headline adds only streamed session cost; cache writes without a 5m key Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/pricing.ts | 3 ++- scripts/harness-version.json | 2 +- scripts/lib/paid-report.ts | 11 +++-------- test/paid-report-census.test.ts | 9 +++++++++ test/paid-report-cost.test.ts | 2 ++ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/pricing.ts b/lib/pricing.ts index 0e59f3690..6b7697790 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -166,7 +166,8 @@ export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | und if (!row) return undefined; const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0; const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens); - const fiveMinute = usage.cache_creation ? count(usage.cache_creation.ephemeral_5m_input_tokens) : count(usage.cache_creation_input_tokens); + const fiveMinute = typeof usage.cache_creation?.ephemeral_5m_input_tokens === 'number' + ? count(usage.cache_creation.ephemeral_5m_input_tokens) : Math.max(0, count(usage.cache_creation_input_tokens) - oneHour); const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1; return (count(usage.input_tokens) * row.input_per_mtok + count(usage.cache_read_input_tokens) * cachedRate diff --git a/scripts/harness-version.json b/scripts/harness-version.json index d2993d593..e225bbbad 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"a4ef799c233c65e9209a4b654360476fdbf51802","since":6,"nonBehavioral":"the trial model comes from the named records only, as before cost_case_id; cost fields only, no verdict, failure class or series key changes"}, + "scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index a4ef799c2..e0d18da4e 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -230,7 +230,7 @@ export interface JUnitCensus { deselected: Map; /** Per shard key: skip reason -> skipped testcase count. */ shardSkipReasons: Map>; - /** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */ + /** Streamed session cost (sessions without a terminal result), which no collector record carries; the headline adds it to the collectors' total_cost_usd. */ ledgerCostUsd: number; /** ` :: ()` for every testcase no rule attributes. */ unattributed: string[]; @@ -253,17 +253,12 @@ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-fl */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; - const unrecorded = unrecordedCosts(records, sessions); + const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded); return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } -/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */ -function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array { - return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); -} - const knownSum = (costs: Array) => costs.reduce((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0); /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ @@ -311,7 +306,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const sessions = caseSessions(ledger, id, byCase.size === 1); const cost = caseCost(id, mine, sessions); - census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions)); + census.ledgerCostUsd += knownSum(sessions.map(s => s.billed === false ? s.streamed_cost_usd : 0)); const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; const failureClass = credentials ? 'infra' : recorded; const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 15f483f99..103e1c8d2 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -137,6 +137,15 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown'); }); + test('a billed session\'s cost is already in its collector total, so the verdict never adds it again (Codex host, unbound label record)', () => { + const r = report('periodic', dir => { + results(dir, 'skill-e2e-cso', [{ name: '/cso full audit', suite: 'cso', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.5 }]); + fs.writeFileSync(path.join(shard(dir, 'skill-e2e-cso'), 'session-ledger.jsonl'), + JSON.stringify({ key: 'k', runner: 'codex', started_at: 't', elapsed_ms: 1, end: 'completed', billed: true, cost_usd: 0.5 }) + '\n'); + }); + expect(collector(r.dir).verdict.costUsd).toBe(0.5); + }); + // Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard // wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class); // fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped; diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index 174d8be38..ac5e08eff 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -23,6 +23,8 @@ describe('pricing usage', () => { .toBeCloseTo(4 + 20 + 0.2 + 5, 6); expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5')) .toBeCloseTo(5 + 16, 6); + expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5')) + .toBeCloseTo(5 + 16, 6); expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined(); }); From ceba1f4cc61d00041caced390725afc8dda756c4 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:59:05 +0000 Subject: [PATCH 08/35] docs: how the paid report prices a trial and what stays cost-unknown Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/TESTING_INTERNALS.md | 51 ++++++++++++++++++++++++++++++--------- docs/evals/census-red.md | 2 +- docs/troubleshooting.md | 21 +++++++++++----- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 31fefa201..eb6592644 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -640,15 +640,36 @@ this checkout to read it)`. samples, rationale}` entry per failing dimension (since EVAL_POLICY v3 `mean` holds the panel's gating median), each part capped at 200 characters. `sessions[]` (at most 32) summarizes the trial's session-ledger -rows. `cost_known: false` marks a record or trial whose harness captured no -billing: PTY and Codex sessions never bill, and an SDK session bills only with a -terminal `total_cost_usd`. It also marks a trial no result record could be -attributed to: a record named by the case id or carrying `case_id` is that -case's, and a record named by a display label (`/qa fix loop`) is the shard's -only case's when the shard ran one case, so a multi-case shard whose records -carry display labels leaves those trials cost-unknown rather than guessing -which case paid (`junitCensus` in `scripts/lib/paid-report.ts`). An absent -field means known. Census and marathon cost lines print the known sum and +rows. `cost_known: false` marks a record or trial whose cost is not fully +known; an absent field means known. A record carries it when its harness +captured no billing (a Codex session that completed no turn or whose model has +no price row). A rule case's trial in the JUnit census (`caseCost` in +`scripts/lib/paid-report.ts`) costs its records' sum plus what no record +carries: each ledger session without a terminal result (`billed: false`) adds +its `streamed_cost_usd`, and a trial with no record adds each billed session's +own `cost_usd` (Codex). The trial is cost-unknown when any record says +`cost_known: false`, when any of those sessions has no priced cost (every PTY +session, a `claude -p` session that ended without a result, an aborted Agent +SDK session with no usage or an unpriced model), or when it has neither records +nor sessions. A case in `MODEL_FREE_CASES` (`timeline-event-flow`, which runs +no model) with neither is a known $0. A panel trial (`behavior`, `judge`) from +an isolated trial shard keeps the older rule (`trialCostKnown` in +`test/helpers/eval-store.ts`): known only with at least one record and every +session billed, and streamed cost is not added. + +Records are attributed to a case this way: a record named by the case id or +carrying `case_id` is that case's, and a record named by a display label +(`/qa fix loop`) is the shard's only case's when the shard ran one case. In a +multi-case shard, a display-label record whose `runSkillTest` test name is a +registered case id carries `cost_case_id` (`recordE2E`), and the census adds +its cost to that case only: the failed record, failure class and model still +come from the records named by case id, so no verdict changes. A display-label +record without `cost_case_id` in a multi-case shard is not attributed rather +than guessed, and its case is cost-unknown when nothing else prices it +(`junitCensus`). The report's headline `cost $` adds the streamed session +cost no collector record carries, so a newly priced trial counts once. + +Census and marathon cost lines print the known sum and `N of M trial(s) cost unknown`, and a train verdict's `ATTEMPT:` line prints `cost=$ known + of case(s) cost unknown` when some but not every case in the train record has a cost (a record with no priced case prints @@ -660,13 +681,21 @@ including `runSharedInteractive`, the PTY session and `runRecordedCodexEval`) appends one best-effort line per armed session to `$GSTACK_EVAL_DIR/session-ledger.jsonl` (`test/helpers/session-ledger.ts`): a stable key `: