diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 5b27d9e0e..22c6c416b 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,7 +13,7 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"}, + "scripts/lib/paid-report.ts": {"blob":"6adbde198309328bae17c3b17d199963e19a4c7e","since":6,"nonBehavioral":"the paid report headline adds session cost no collector record carries; cost fields only, no verdict or failure class changes"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index b00d662e5..6adbde198 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -230,6 +230,8 @@ export interface JUnitCensus { deselected: Map; /** Per shard key: skip reason -> skipped testcase count. */ shardSkipReasons: Map>; + /** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */ + ledgerCostUsd: number; /** ` :: ()` for every testcase no rule attributes. */ unattributed: string[]; } @@ -251,13 +253,19 @@ export const MODEL_FREE_CASES: ReadonlySet = new Set(['timeline-event-fl */ export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } { if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false }; - const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); + const unrecorded = unrecordedCosts(records, sessions); const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c)); - const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))] - .reduce((sum, c) => sum + c, 0); + const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded); return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) }; } +/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */ +function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array { + return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd); +} + +const knownSum = (costs: Array) => costs.reduce((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0); + /** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */ /** The red-census guide the report and every eval:pass-rates view link. */ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; @@ -269,7 +277,8 @@ export function sliceArtifactName(root: string, slice: number, attempt: number): export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, base: Record, cliVersion?: string): JUnitCensus { - const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; + const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [], + ledgerCostUsd: 0 }; const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry])); for (const { root, result } of artifacts) { for (const outcome of result.outcomes) { @@ -302,6 +311,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const sessions = caseSessions(ledger, id, byCase.size === 1); const cost = caseCost(id, mine, sessions); + census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions)); const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; const failureClass = credentials ? 'infra' : recorded; const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; @@ -796,7 +806,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...(spend.executions ? { reruns: spend } : {}), actionRequired: verdict.problems.length, wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, - costUsd: Math.round(costUsd * 100) / 100, + costUsd: Math.round((costUsd + census.ledgerCostUsd) * 100) / 100, redispatchEligible: red && infraOnly(verdict.problems), }; const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped'); diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index ff9dbf168..15f483f99 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -123,6 +123,20 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr expect(collector(r.dir).verdict.costUsd).toBe(0.75); }); + test('a record-less trial\'s streamed session cost reaches the trial and the verdict total once (GSTA-150 outside voice: $0.65 known, headline $0.00)', () => { + const ledger = (dir: string, slug: string, costs: number[]) => fs.writeFileSync(path.join(shard(dir, slug), 'session-ledger.jsonl'), + costs.map(cost => JSON.stringify({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', billed: false, streamed_cost_usd: cost })).join('\n') + '\n'); + const r = report('periodic', dir => { + ledger(dir, 'skill-e2e-qa-workflow--qa-fix-loop', [0.4, 0.25]); + results(dir, 'skill-e2e-benchmark-providers', [{ name: 'benchmark-providers-live', suite: 's', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.1 }]); + ledger(dir, 'skill-e2e-benchmark-providers', [0.5]); + }); + expect(r.history.find(record => record.case === 'qa-fix-loop')).toMatchObject({ outcome: 'passed', cost_usd: 0.65 }); + expect(r.history.find(record => record.case === 'benchmark-providers-live')).toMatchObject({ cost_usd: 0.6 }); + expect(collector(r.dir).verdict.costUsd).toBe(1.25); + expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown'); + }); + // Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard // wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class); // fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped;