Attribute every paid trial's cost: bind label-named records by testName, price aborted SDK sessions from streamed usage, model-free cases cost $0

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-29 04:27:50 +00:00
1 parent 4729f4752b
commit d59225dd74
9 files changed
+204 -12

No files matched your search

+3 -1
View File
@@ -37,6 +37,7 @@ import * as fs from 'fs';
import * as path from 'path';
import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin';
import { resolveEvalModel } from '../../lib/eval-model';
import { streamedUsageCostUsd } from '../../lib/pricing';
import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger';
import { hermeticChildEnv } from './hermetic-env';
import type { SkillTestResult } from './session-runner';
@@ -353,11 +354,12 @@ export async function runAgentSdkTest(
const budget = typeof opts.sessionBudgetMs === 'function' ? opts.sessionBudgetMs() : opts.sessionBudgetMs;
const aborted = controller.signal.aborted || opts.signal?.aborted === true;
const structured = observer.verdict();
const streamed = observer.billed ? undefined : streamedUsageCostUsd(assistantTurns, model);
const end: SessionEnd = aborted ? (budget !== undefined && elapsed + 1000 >= budget ? 'session_timeout' : 'aborted') : structured?.end ?? fallback;
appendSessionLedger({ key: sessionKey('agent-sdk', opts.testName), ...(opts.testName ? { test_name: opts.testName } : {}), runner: 'agent-sdk',
started_at: startedAt, ...(budget !== undefined ? { budget_ms: budget } : {}), elapsed_ms: elapsed, end,
...(end === 'session_timeout' ? { evidence: `armed ${budget}ms session timeout fired` } : structured && !aborted ? { evidence: structured.evidence } : {}),
liveness: observer.summary(performance.now()), billed: observer.billed });
liveness: observer.summary(performance.now()), billed: observer.billed, ...(streamed !== undefined ? { streamed_cost_usd: streamed } : {}) });
};
const controller = new AbortController();
let activeQuery: ReturnType<QueryProvider> | undefined;
+1
View File
@@ -274,6 +274,7 @@ export function recordE2E(
duration_ms: result.duration,
cost_usd: result.costEstimate.estimatedCost,
...(result.costKnown === false ? { cost_known: false } : {}),
...(result.testName && result.testName !== name && result.testName in E2E_TIERS ? { cost_case_id: result.testName } : {}),
...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}),
transcript: result.transcript,
output: result.output?.slice(0, 2000),
+3
View File
@@ -86,6 +86,9 @@ export interface EvalTestEntry {
// TRIAL_ENV variables the paid runner sets on an isolated trial shard.
/** Registry id (E2E_TIERS / LLM_JUDGE_TOUCHFILES key) this record belongs to. */
case_id?: string;
/** Registry id whose cost this record carries (runSkillTest's testName) when its name is a display label;
* the report binds cost by it, never the failed record or its failure class. */
cost_case_id?: string;
kind?: EvalCaseKind;
/** 1-based trial index within the case's panel. */
trial?: number;
+2
View File
@@ -52,6 +52,8 @@ export interface SessionLedgerRow {
liveness?: LivenessSummary;
/** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */
billed?: boolean;
/** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */
streamed_cost_usd?: number;
/** Codex sessions: the model, its billed cost and token classes. */
model?: string;
cost_usd?: number;
+3 -1
View File
@@ -61,6 +61,8 @@ export interface SkillTestResult {
tokenUsage?: CodexTokenUsage;
/** Codex host: false when the session completed no turn or its model has no price row. */
costKnown?: boolean;
/** runSkillTest's testName: the registered case id the session ledger names, which recordE2E binds the record's cost to. */
testName?: string;
}
/** Local default startup grace: 90s covers observed API queue latency
@@ -741,5 +743,5 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af
turnsUsed,
};
return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs };
return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs, ...(testName ? { testName } : {}) };
}
+103
View File
@@ -0,0 +1,103 @@
/**
* PR 2 of GSTA-150: every trial's cost is attributed. A display-label record binds its cost by
* runSkillTest's testName (cost_case_id) without changing the failed record or failure class; a case
* with no record takes its sessions' streamed cost; a listed model-free case costs a known $0.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { MODEL_FREE_CASES, caseCost, junitCensus } from '../scripts/lib/paid-report';
import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing';
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
import type { SessionLedgerRow } from './helpers/session-ledger';
const ROOT = path.resolve(import.meta.dir, '..');
const session = (over: Partial<SessionLedgerRow>): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over });
describe('pricing usage', () => {
test('cache writes bill 1.25x input (5-minute) and 2x (1-hour); reads at the cached rate', () => {
// claude-opus-5-5: input $4, output $20, cache read $0.20 per Mtok.
expect(anthropicUsageCostUsd({ input_tokens: 1_000_000, output_tokens: 1_000_000, cache_read_input_tokens: 1_000_000, cache_creation_input_tokens: 1_000_000 }, 'claude-opus-5-5'))
.toBeCloseTo(4 + 20 + 0.2 + 5, 6);
expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5'))
.toBeCloseTo(5 + 16, 6);
expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined();
});
test('streamed messages count once per id at their largest output, sidechains included; an unpriced model is unknown', () => {
const msg = (id: string, output: number, extra: Record<string, unknown> = {}) => ({ message: { id, model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: output } }, ...extra });
expect(streamedUsageCostUsd([msg('a', 10), msg('a', 1_000_000), msg('b', 500_000, { parent_tool_use_id: 'tool-1' })])).toBeCloseTo(30, 6);
expect(streamedUsageCostUsd([{ message: { id: 'a', model: '<synthetic>', usage: { output_tokens: 5 } } }])).toBeUndefined();
expect(streamedUsageCostUsd([{ message: { id: 'a', model: 'no-such-model', usage: { output_tokens: 5 } } }, msg('b', 5)])).toBeUndefined();
expect(streamedUsageCostUsd([{ message: { id: 'a', usage: { output_tokens: 1_000_000 } } }], 'claude-opus-5-5')).toBeCloseTo(20, 6);
});
});
describe('caseCost', () => {
test('records decide when present; sessions never add to them', () => {
expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 });
expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false });
});
test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => {
expect(caseCost('auq', [], [session({ billed: false, streamed_cost_usd: 0.123 })])).toEqual({ cost_usd: 0.12 });
expect(caseCost('auq', [], [session({ billed: false })])).toEqual({ cost_usd: 0, cost_known: false });
expect(caseCost('auq', [], [session({ runner: 'claude-p', billed: true })])).toEqual({ cost_usd: 0, cost_known: false });
});
test('only a listed model-free case with neither record nor session costs a known $0', () => {
expect(caseCost('timeline-event-flow', [], [])).toEqual({ cost_usd: 0 });
expect(caseCost('some-other-case', [], [])).toEqual({ cost_usd: 0, cost_known: false });
});
test('every listed model-free case body starts no model session', () => {
for (const id of MODEL_FREE_CASES) {
const file = E2E_TOUCHFILES[id]?.find(entry => /^test\/[^/]+\.test\.ts$/.test(entry));
expect(file, id).toBeTruthy();
const source = fs.readFileSync(path.join(ROOT, file!), 'utf8');
const start = source.search(new RegExp(`IfSelected\\(\\s*['"\`]${id}['"\`]`));
expect(start, id).toBeGreaterThan(-1);
const rest = source.slice(start + 1);
const end = rest.search(/IfSelected\(/);
const body = end === -1 ? rest : rest.slice(0, end);
expect(body, id).not.toMatch(/runSkillTest|runAgentSdkTest|runCodex|launchClaudePty|runPtySession|claude-pty|callJudge|judge\w*\(/);
}
});
});
describe('junitCensus cost binding', () => {
const key = 'test/skill-e2e-bws.test.ts';
const census = (records: any[], ledger: SessionLedgerRow[] = []) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cost-'));
const shard = path.join(root, 'shards', 'skill-e2e-bws');
fs.mkdirSync(shard, { recursive: true });
fs.writeFileSync(path.join(shard, 'junit.xml'), `<testsuites><testsuite name="${key}">
<testcase name="browse-basic" classname="${key}" time="1"></testcase>
<testcase name="browse-snapshot" classname="${key}" time="1"><failure type="AssertionError" message="expected snapshot"/></testcase>
</testsuite></testsuites>`);
fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), ledger.map(row => JSON.stringify(row)).join('\n'));
const manifest: any = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, profile: 'full', selection: { e2e: null, judges: null }, entries: [{ file: key, slice: 1, status: 'planned', estimatedMs: 1 }] };
const result: any = { version: 1, tier: 'gate', profile: 'full', selection: { e2e: null, judges: null }, sliceIndex: 1, sliceCount: 1, attempt: 1,
outcomes: [{ files: [key], status: 'failed', exitCode: 1, elapsedMs: 10, executedTests: 2, skippedTests: 0 }] };
try { return junitCensus(manifest, [{ root, result }], new Map([[key, records]]), {}); } finally { fs.rmSync(root, { recursive: true, force: true }); }
};
const labelled = [
{ name: 'browse basic commands', passed: true, cost_usd: 0.05, cost_case_id: 'browse-basic' },
{ name: 'browse snapshot flags', passed: false, cost_usd: 0.06, cost_case_id: 'browse-snapshot', exit_reason: 'timeout', failure_class: 'infra', error: 'record error' },
];
test('display-label records bind their cost by cost_case_id in a multi-case shard', () => {
const cases = census(labelled).ruleCases;
expect(cases.map(c => [c.id, c.costUsd, c.costKnown])).toEqual([['browse-basic', 0.05, true], ['browse-snapshot', 0.06, true]]);
});
test('the failed record and failure class stay what the named records give, as without cost_case_id', () => {
const bound = census(labelled).ruleCases.find(c => c.id === 'browse-snapshot')!;
const unbound = census(labelled.map(({ cost_case_id, ...rest }) => rest)).ruleCases.find(c => c.id === 'browse-snapshot')!;
expect(unbound).toMatchObject({ costUsd: 0, costKnown: false });
expect({ failureClass: bound.failureClass, error: bound.error, line: bound.line?.replace(/ {2}\[slice.*$/, '') })
.toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') });
expect(bound.failureClass).toBe('assertion');
});
});