From 9b64b900c0458c849fb2afa250328e9f46b0c76d Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 17:57:55 +0000 Subject: [PATCH] F: cost_known: a trial whose harness captured no billing says so; totals never hide it cost_usd stays a finite number. Trial and eval records gain an optional cost_known boolean (absent means known, so old records still parse; the validator accepts only a boolean). Session-ledger rows record whether the session's terminal result carried total_cost_usd; PTY and Codex sessions never do, and SDK captures without a terminal result do not. A trial is cost-known only when it has eval records and every record and session captured billing, so JUnit-only PTY cases (125 of 298 trials in run 37198445662 recorded 0) now read cost unknown. The census headline prints the known sum plus the unknown count, never a bare total over unknown costs, and no total when every cost is unknown; the history import keeps the flag. PTY transcripts are not priced: test/helpers/pricing.ts estimates exclude cache writes, so they cannot give the billed cost. --- scripts/eval-flake-rank.ts | 2 +- scripts/lib/paid-report.ts | 19 +++++-- scripts/test-paid-shards.ts | 4 +- test/cost-known.test.ts | 95 ++++++++++++++++++++++++++++++++ test/helpers/agent-sdk-runner.ts | 2 +- test/helpers/codex-eval.ts | 4 +- test/helpers/eval-store.ts | 13 +++++ test/helpers/pty/launch.ts | 2 +- test/helpers/pty/session.ts | 2 +- test/helpers/session-ledger.ts | 5 ++ test/helpers/session-runner.ts | 2 +- 11 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 test/cost-known.test.ts diff --git a/scripts/eval-flake-rank.ts b/scripts/eval-flake-rank.ts index de8e17664..8ce5d746d 100644 --- a/scripts/eval-flake-rank.ts +++ b/scripts/eval-flake-rank.ts @@ -265,7 +265,7 @@ export function backfillEvalFiles(files: string[], run?: { run_id: string; sha?: tier: caseTier(id, registry), kind: registry.kinds[id]!, trial: 1, panel: { n: 1, k: 1 }, attempt: 1, outcome, ...(outcome === 'failed' ? { failure_class: failureClassOf(entry) } : {}), exit_reason: entry.exit_reason, error: sanitizeTrialError(entry.error), - duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0), + duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0), ...(entry.cost_known === false ? { cost_known: false } : {}), model: entry.model, cli_version: result.claude_cli_version, policy_version: 0, quarantined: false, execution: entry.execution === 'reused' ? 'reused' : 'executed', source: 'backfill', run_id: run?.run_id ?? `local:${file}`, sha: run?.sha ?? result.git_sha, recorded_at: run?.timestamp ?? result.timestamp, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 4174b3ef1..e460e78ad 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -33,7 +33,7 @@ import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, - trialFailureFields, trialSessions, type TrialFailureDetail, + trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; import { publishedFence, sanitizePublishedText } from './published-text'; @@ -319,7 +319,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), - cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, + cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions), ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); } @@ -482,7 +482,13 @@ export interface ReportHeadline { redispatchEligible: boolean; } -export function formatHeadline(h: ReportHeadline): string[] { +/** Known cost, never a bare total when any trial's cost is unknown, and no total when none is known. */ +export function formatCost(knownUsd: number, unknownTrials: number, trials: number): string { + if (trials > 0 && unknownTrials === trials) return `cost unknown (no billing captured for any of ${trials} trial(s))`; + return `cost $${knownUsd.toFixed(2)}${unknownTrials ? ` known + ${unknownTrials} of ${trials} trial(s) cost unknown` : ''}`; +} + +export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number; trials: number } = { unknownTrials: 0, trials: 0 }): string[] { const c = h.counts; const minutes = h.wallMs === null ? 'unknown' : `${Math.floor(h.wallMs / 60_000)}m${String(Math.round((h.wallMs % 60_000) / 1000)).padStart(2, '0')}s`; return [ @@ -490,7 +496,7 @@ export function formatHeadline(h: ReportHeadline): string[] { ` rule ${c.rule.passed}/${c.rule.total} · behavior ${c.behavior.passed}/${c.behavior.total}${c.behavior.split ? ` (${c.behavior.split} split)` : ''}` + ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`, ` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`, - ` wall ${minutes} · cost $${h.costUsd.toFixed(2)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`, + ` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`, ]; } @@ -675,7 +681,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}), ...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}), - duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}), + duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.cost_known === false ? { cost_known: false } : {}), ...(t.model ? { model: t.model } : {}), ...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}), quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); } @@ -742,7 +748,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo costUsd: Math.round(costUsd * 100) / 100, redispatchEligible: red && infraOnly(verdict.problems), }; - const headlineLines = formatHeadline(headline); + const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped'); + const headlineLines = formatHeadline(headline, { unknownTrials: executed.filter(r => r.cost_known === false).length, trials: executed.length }); for (const line of headlineLines) console.log(line); if (failureLines.length) { console.log('[test:paid] failures and split verdicts:'); diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 8fcecd0ee..24882ab4b 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -83,7 +83,7 @@ import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, - trialFailureFields, trialSessions, type TrialFailureCause, type TrialFailureDetail, type TrialSessionSummary, + trialCostKnown, trialFailureFields, trialSessions, type TrialFailureCause, type TrialFailureDetail, type TrialSessionSummary, } from '../test/helpers/eval-store'; import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger'; import { E2E_KINDS } from '../test/helpers/touchfiles-data'; @@ -726,6 +726,7 @@ export interface ShardTrialRecord { failure_cause_evidence?: string; failure_detail?: TrialFailureDetail; sessions?: TrialSessionSummary[]; + cost_known?: false; } type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] }; @@ -759,6 +760,7 @@ export function classifyTrialShard( cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100, duration_ms: outcome.elapsedMs, ...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []), + ...trialCostKnown(evidence.records, evidence.sessions), }; const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => { const cls = evidence.contract !== null ? 'contract' : failureClass; diff --git a/test/cost-known.test.ts b/test/cost-known.test.ts new file mode 100644 index 000000000..92bd223c0 --- /dev/null +++ b/test/cost-known.test.ts @@ -0,0 +1,95 @@ +/** + * cost_known (F): cost_usd stays a finite number; a trial whose harness + * captured no billing says so, and totals never print a bare sum over + * unknown costs. Old records without the field still read as known. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { TRIAL_OUTCOME_SCHEMA, formatTrialOutcomes, parseTrialOutcomes, trialCostKnown, type TrialOutcomeRecord } from './helpers/eval-store'; +import { SessionObserver } from './helpers/session-ledger'; +import { runRecordedCodexEval } from './helpers/codex-eval'; +import { trialRecordProblems as trialRecordProblemsV1 } from './fixtures/trial-record-v1-reader'; +import { backfillEvalFiles, type Registry } from '../scripts/eval-flake-rank'; +import { classifyTrialShard, formatCost, runPaidReport } from '../scripts/test-paid-shards'; + +const record = (over: Partial = {}): TrialOutcomeRecord => ({ + schema: TRIAL_OUTCOME_SCHEMA, case: 'c', file: 'test/c.test.ts', tier: 'gate', kind: 'rule', trial: 1, panel: { n: 1, k: 1 }, attempt: 1, + outcome: 'passed', duration_ms: 1, cost_usd: 0, policy_version: 1, quarantined: false, execution: 'executed', source: 'junit', ...over, +}); + +describe('producers', () => { + test('a session is billed only when its terminal result carries total_cost_usd', () => { + const billed = new SessionObserver(0); + billed.observe({ type: 'result', subtype: 'success', total_cost_usd: 0.42 }, 1); + const unbilled = new SessionObserver(0); + unbilled.observe({ type: 'assistant', message: { content: [] } }, 1); + expect([billed.billed, unbilled.billed]).toEqual([true, false]); + }); + + test('a trial is cost-known only when every record and session captured billing', () => { + expect(trialCostKnown([{ cost_known: true }, {}], [{ billed: true }])).toEqual({}); + expect(trialCostKnown([{}], [{ billed: false }])).toEqual({ cost_known: false }); + expect(trialCostKnown([{ cost_known: false }])).toEqual({ cost_known: false }); + expect(trialCostKnown([], [{ billed: true }])).toEqual({ cost_known: false }); + }); + + test('Codex records and an isolated trial without billing say cost unknown; cost_usd stays a number', async () => { + const entries: any[] = []; + await runRecordedCodexEval({ name: 'n', suite: 's', budgetMs: 1_000, record: entry => entries.push(entry), validate: () => {}, + run: async () => ({ output: '', stderr: '', exitCode: 0, durationMs: 5, toolCalls: [], tokens: 10, rawLines: [] } as any) }); + expect(entries[0]).toMatchObject({ cost_usd: 0, cost_known: false }); + const plan = { kind: 'behavior' as const, panel: { n: 3, k: 2 }, quarantined: false }; + const shard = { status: 'passed' as const, executedTests: 1, skippedTests: 0, elapsedMs: 5 }; + const known = classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: true, cost_usd: 0.3 }], contract: null, sessions: [] }); + const pty = classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: true, cost_usd: 0 }], contract: null, + sessions: [{ key: 'pty:c#1', runner: 'pty', started_at: 't', elapsed_ms: 1, end: 'completed', billed: false }] }); + expect(known).toMatchObject({ cost_usd: 0.3 }); + expect(known.cost_known).toBeUndefined(); + expect(pty).toMatchObject({ cost_usd: 0, cost_known: false }); + }); +}); + +describe('serializer and readers', () => { + test('mixed old and new records round-trip; the frozen v1 reader accepts the new field', () => { + const records = [record(), record({ trial: 1, case: 'd', cost_known: false }), record({ case: 'e', cost_known: true })]; + expect(parseTrialOutcomes(formatTrialOutcomes(records))).toEqual({ records, errors: [] }); + for (const r of records) expect(trialRecordProblemsV1(r)).toEqual([]); + }); + + test('history import keeps an unknown cost; an old legacy record stays known', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cost-known-backfill-')); + const file = path.join(dir, '1.91.19-x-e2e-2026-10-04-1200.json'); + fs.writeFileSync(file, JSON.stringify({ tier: 'e2e', git_sha: 's', timestamp: 't', tests: [ + { name: 'old-case', passed: true, cost_usd: 1.5, duration_ms: 1 }, { name: 'new-case', passed: true, cost_usd: 0, cost_known: false, duration_ms: 1 }] })); + const registry: Registry = { tiers: { 'old-case': 'gate', 'new-case': 'gate' }, kinds: { 'old-case': 'rule', 'new-case': 'rule' }, + touchfiles: {}, judgeTouchfiles: {}, globals: [], testNames: {} }; + const { records } = backfillEvalFiles([file], undefined, registry); + fs.rmSync(dir, { recursive: true, force: true }); + expect(records.find(r => r.case === 'old-case')).not.toHaveProperty('cost_known'); + expect(records.find(r => r.case === 'new-case')).toMatchObject({ cost_usd: 0, cost_known: false }); + }); + + test('totals: known sum plus the unknown count; never a bare total over unknowns; no total when all are unknown', () => { + expect(formatCost(4.2, 0, 10)).toBe('cost $4.20'); + expect(formatCost(4.2, 3, 10)).toBe('cost $4.20 known + 3 of 10 trial(s) cost unknown'); + expect(formatCost(0, 10, 10)).toBe('cost unknown (no billing captured for any of 10 trial(s))'); + }); + + test('the census report headline counts trials whose cost is unknown', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cost-known-report-')); + fs.cpSync(path.join(import.meta.dir, 'fixtures', 'paid-report-census', 'gate'), dir, { recursive: true }); + const lines: string[] = []; + const { log, error } = console; + console.log = (...args: unknown[]) => { lines.push(args.join(' ')); }; + console.error = console.log; + try { runPaidReport(dir, { env: {} }); } finally { console.log = log; console.error = error; } + const history = fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); + fs.rmSync(dir, { recursive: true, force: true }); + // The 2026-10-03 gate fixture holds JUnit results only: no eval record captured billing. + expect(history.every(r => r.cost_known === false && r.cost_usd === 0)).toBe(true); + expect(lines.join('\n')).toContain(`cost unknown (no billing captured for any of ${history.length} trial(s))`); + expect(lines.join('\n')).not.toContain('cost $0.00'); + }); +}); diff --git a/test/helpers/agent-sdk-runner.ts b/test/helpers/agent-sdk-runner.ts index 870d0652b..210bba92d 100644 --- a/test/helpers/agent-sdk-runner.ts +++ b/test/helpers/agent-sdk-runner.ts @@ -357,7 +357,7 @@ export async function runAgentSdkTest( appendSessionLedger({ key: sessionKey('agent-sdk', opts.testName), ...(opts.testName ? { test_name: opts.testName } : {}), runner: 'agent-sdk', started_at: startedAt, ...(budget !== undefined ? { budget_ms: budget } : {}), elapsed_ms: elapsed, end, ...(end === 'session_timeout' ? { evidence: `armed ${budget}ms session timeout fired` } : structured && !aborted ? { evidence: structured.evidence } : {}), - liveness: observer.summary(performance.now()) }); + liveness: observer.summary(performance.now()), billed: observer.billed }); }; const controller = new AbortController(); let activeQuery: ReturnType | undefined; diff --git a/test/helpers/codex-eval.ts b/test/helpers/codex-eval.ts index e49f15348..1a402da51 100644 --- a/test/helpers/codex-eval.ts +++ b/test/helpers/codex-eval.ts @@ -201,7 +201,7 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise, sessions: ReadonlyArray<{ billed?: boolean }> = []): { cost_known?: false } { + return records.length > 0 && records.every((r) => r.cost_known !== false) && sessions.every((s) => s.billed !== false) ? {} : { cost_known: false }; +} + /** Cause, cause evidence and detail of one failed trial, for both record builders. */ export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): { failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail; diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index 38668fb26..4ef0bcb7b 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -242,7 +242,7 @@ export async function launchClaudePty( recorded = true; const elapsed = performance.now() - startedMono; appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: timeoutMs, - elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...idlePanelEnd(handle.visibleText()) }); + elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...idlePanelEnd(handle.visibleText()), billed: false }); } return close(); } }); diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 33e4e8890..11bd57473 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -92,7 +92,7 @@ export async function runPtySession(plan: PtySessionPlan): Promise { } finally { appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started, - end, ...idlePanelEnd(session.visibleText()) }); + end, ...idlePanelEnd(session.visibleText()), billed: false }); try { try { plan.beforeClose?.(session, failure); } catch (error) { if (!failure) { failure = { error }; throw error; } } diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index bd091fadc..6e3c9e837 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -46,6 +46,8 @@ export interface SessionLedgerRow { end: SessionEnd; evidence?: string; liveness?: LivenessSummary; + /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ + billed?: boolean; } /** One line, control characters stripped, mentions neutralized, capped. */ @@ -118,6 +120,8 @@ export class SessionObserver { private refusal?: string; private apiError?: string; private phraseRefusal?: string; + /** A terminal result carried total_cost_usd. */ + billed = false; constructor(private readonly startedAt: number) { this.lastAt = startedAt; } @@ -167,6 +171,7 @@ export class SessionObserver { if (subtype === 'model_refusal_no_fallback') this.refusal ??= this.refusalLine(subtype, event.api_refusal_category); } else if (event.type === 'result') { this.requesting = false; + if (typeof event.total_cost_usd === 'number' && Number.isFinite(event.total_cost_usd)) this.billed = true; if (event.stop_reason === 'refusal') this.refusal ??= this.refusalLine('result stop_reason refusal', undefined); if (event.is_error === true && typeof event.result === 'string') this.noteResultText(event.result); } diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index f2273626d..d40eb29e8 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -654,7 +654,7 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af started_at: startedAt, budget_ms: timeout, elapsed_ms: performance.now() - startMono, end, ...(timedOut ? { evidence: timedOutInStartup ? `no output within the ${startupGraceMs}ms startup grace` : `armed ${timeout}ms session timeout fired` } : structured ? { evidence: structured.evidence } : {}), - liveness: observer.summary(performance.now()) }); + liveness: observer.summary(performance.now()), billed: observer.billed }); // Parse all collected NDJSON lines const parsed = parseNDJSON(collectedLines);