mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-21 07:33:14 +01:00
Attribute every paid trial's cost: bind label-named records by testName, price aborted SDK sessions from streamed usage, model-free cases cost $0
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
4729f4752b
commit
d59225dd74
9 files changed
+204
-12
No files matched your search
@@ -37,6 +37,7 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin';
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
import { streamedUsageCostUsd } from '../../lib/pricing';
|
||||
import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import type { SkillTestResult } from './session-runner';
|
||||
@@ -353,11 +354,12 @@ export async function runAgentSdkTest(
|
||||
const budget = typeof opts.sessionBudgetMs === 'function' ? opts.sessionBudgetMs() : opts.sessionBudgetMs;
|
||||
const aborted = controller.signal.aborted || opts.signal?.aborted === true;
|
||||
const structured = observer.verdict();
|
||||
const streamed = observer.billed ? undefined : streamedUsageCostUsd(assistantTurns, model);
|
||||
const end: SessionEnd = aborted ? (budget !== undefined && elapsed + 1000 >= budget ? 'session_timeout' : 'aborted') : structured?.end ?? fallback;
|
||||
appendSessionLedger({ key: sessionKey('agent-sdk', opts.testName), ...(opts.testName ? { test_name: opts.testName } : {}), runner: 'agent-sdk',
|
||||
started_at: startedAt, ...(budget !== undefined ? { budget_ms: budget } : {}), elapsed_ms: elapsed, end,
|
||||
...(end === 'session_timeout' ? { evidence: `armed ${budget}ms session timeout fired` } : structured && !aborted ? { evidence: structured.evidence } : {}),
|
||||
liveness: observer.summary(performance.now()), billed: observer.billed });
|
||||
liveness: observer.summary(performance.now()), billed: observer.billed, ...(streamed !== undefined ? { streamed_cost_usd: streamed } : {}) });
|
||||
};
|
||||
const controller = new AbortController();
|
||||
let activeQuery: ReturnType<QueryProvider> | undefined;
|
||||
|
||||
@@ -274,6 +274,7 @@ export function recordE2E(
|
||||
duration_ms: result.duration,
|
||||
cost_usd: result.costEstimate.estimatedCost,
|
||||
...(result.costKnown === false ? { cost_known: false } : {}),
|
||||
...(result.testName && result.testName !== name && result.testName in E2E_TIERS ? { cost_case_id: result.testName } : {}),
|
||||
...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}),
|
||||
transcript: result.transcript,
|
||||
output: result.output?.slice(0, 2000),
|
||||
|
||||
@@ -86,6 +86,9 @@ export interface EvalTestEntry {
|
||||
// TRIAL_ENV variables the paid runner sets on an isolated trial shard.
|
||||
/** Registry id (E2E_TIERS / LLM_JUDGE_TOUCHFILES key) this record belongs to. */
|
||||
case_id?: string;
|
||||
/** Registry id whose cost this record carries (runSkillTest's testName) when its name is a display label;
|
||||
* the report binds cost by it, never the failed record or its failure class. */
|
||||
cost_case_id?: string;
|
||||
kind?: EvalCaseKind;
|
||||
/** 1-based trial index within the case's panel. */
|
||||
trial?: number;
|
||||
|
||||
@@ -52,6 +52,8 @@ export interface SessionLedgerRow {
|
||||
liveness?: LivenessSummary;
|
||||
/** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */
|
||||
billed?: boolean;
|
||||
/** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */
|
||||
streamed_cost_usd?: number;
|
||||
/** Codex sessions: the model, its billed cost and token classes. */
|
||||
model?: string;
|
||||
cost_usd?: number;
|
||||
|
||||
@@ -61,6 +61,8 @@ export interface SkillTestResult {
|
||||
tokenUsage?: CodexTokenUsage;
|
||||
/** Codex host: false when the session completed no turn or its model has no price row. */
|
||||
costKnown?: boolean;
|
||||
/** runSkillTest's testName: the registered case id the session ledger names, which recordE2E binds the record's cost to. */
|
||||
testName?: string;
|
||||
}
|
||||
|
||||
/** Local default startup grace: 90s covers observed API queue latency
|
||||
@@ -741,5 +743,5 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af
|
||||
turnsUsed,
|
||||
};
|
||||
|
||||
return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs };
|
||||
return { toolCalls, browseErrors, exitReason, duration, output: resultLine?.result || '', costEstimate, transcript, model, firstResponseMs, maxInterTurnMs, ...(testName ? { testName } : {}) };
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* PR 2 of GSTA-150: every trial's cost is attributed. A display-label record binds its cost by
|
||||
* runSkillTest's testName (cost_case_id) without changing the failed record or failure class; a case
|
||||
* with no record takes its sessions' streamed cost; a listed model-free case costs a known $0.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { MODEL_FREE_CASES, caseCost, junitCensus } from '../scripts/lib/paid-report';
|
||||
import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import type { SessionLedgerRow } from './helpers/session-ledger';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const session = (over: Partial<SessionLedgerRow>): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over });
|
||||
|
||||
describe('pricing usage', () => {
|
||||
test('cache writes bill 1.25x input (5-minute) and 2x (1-hour); reads at the cached rate', () => {
|
||||
// claude-opus-5-5: input $4, output $20, cache read $0.20 per Mtok.
|
||||
expect(anthropicUsageCostUsd({ input_tokens: 1_000_000, output_tokens: 1_000_000, cache_read_input_tokens: 1_000_000, cache_creation_input_tokens: 1_000_000 }, 'claude-opus-5-5'))
|
||||
.toBeCloseTo(4 + 20 + 0.2 + 5, 6);
|
||||
expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5'))
|
||||
.toBeCloseTo(5 + 16, 6);
|
||||
expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('streamed messages count once per id at their largest output, sidechains included; an unpriced model is unknown', () => {
|
||||
const msg = (id: string, output: number, extra: Record<string, unknown> = {}) => ({ message: { id, model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: output } }, ...extra });
|
||||
expect(streamedUsageCostUsd([msg('a', 10), msg('a', 1_000_000), msg('b', 500_000, { parent_tool_use_id: 'tool-1' })])).toBeCloseTo(30, 6);
|
||||
expect(streamedUsageCostUsd([{ message: { id: 'a', model: '<synthetic>', usage: { output_tokens: 5 } } }])).toBeUndefined();
|
||||
expect(streamedUsageCostUsd([{ message: { id: 'a', model: 'no-such-model', usage: { output_tokens: 5 } } }, msg('b', 5)])).toBeUndefined();
|
||||
expect(streamedUsageCostUsd([{ message: { id: 'a', usage: { output_tokens: 1_000_000 } } }], 'claude-opus-5-5')).toBeCloseTo(20, 6);
|
||||
});
|
||||
});
|
||||
|
||||
describe('caseCost', () => {
|
||||
test('records decide when present; sessions never add to them', () => {
|
||||
expect(caseCost('x', [{ cost_usd: 0.25 }, { cost_usd: 0.1 }], [session({ billed: true, cost_usd: 9 })])).toEqual({ cost_usd: 0.35 });
|
||||
expect(caseCost('x', [{ cost_usd: 0 }], [session({ billed: false, streamed_cost_usd: 0.4 })])).toEqual({ cost_usd: 0, cost_known: false });
|
||||
});
|
||||
|
||||
test('a case with no record takes its sessions\' billed or streamed cost, else stays unknown', () => {
|
||||
expect(caseCost('auq', [], [session({ billed: false, streamed_cost_usd: 0.123 })])).toEqual({ cost_usd: 0.12 });
|
||||
expect(caseCost('auq', [], [session({ billed: false })])).toEqual({ cost_usd: 0, cost_known: false });
|
||||
expect(caseCost('auq', [], [session({ runner: 'claude-p', billed: true })])).toEqual({ cost_usd: 0, cost_known: false });
|
||||
});
|
||||
|
||||
test('only a listed model-free case with neither record nor session costs a known $0', () => {
|
||||
expect(caseCost('timeline-event-flow', [], [])).toEqual({ cost_usd: 0 });
|
||||
expect(caseCost('some-other-case', [], [])).toEqual({ cost_usd: 0, cost_known: false });
|
||||
});
|
||||
|
||||
test('every listed model-free case body starts no model session', () => {
|
||||
for (const id of MODEL_FREE_CASES) {
|
||||
const file = E2E_TOUCHFILES[id]?.find(entry => /^test\/[^/]+\.test\.ts$/.test(entry));
|
||||
expect(file, id).toBeTruthy();
|
||||
const source = fs.readFileSync(path.join(ROOT, file!), 'utf8');
|
||||
const start = source.search(new RegExp(`IfSelected\\(\\s*['"\`]${id}['"\`]`));
|
||||
expect(start, id).toBeGreaterThan(-1);
|
||||
const rest = source.slice(start + 1);
|
||||
const end = rest.search(/IfSelected\(/);
|
||||
const body = end === -1 ? rest : rest.slice(0, end);
|
||||
expect(body, id).not.toMatch(/runSkillTest|runAgentSdkTest|runCodex|launchClaudePty|runPtySession|claude-pty|callJudge|judge\w*\(/);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('junitCensus cost binding', () => {
|
||||
const key = 'test/skill-e2e-bws.test.ts';
|
||||
const census = (records: any[], ledger: SessionLedgerRow[] = []) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-report-cost-'));
|
||||
const shard = path.join(root, 'shards', 'skill-e2e-bws');
|
||||
fs.mkdirSync(shard, { recursive: true });
|
||||
fs.writeFileSync(path.join(shard, 'junit.xml'), `<testsuites><testsuite name="${key}">
|
||||
<testcase name="browse-basic" classname="${key}" time="1"></testcase>
|
||||
<testcase name="browse-snapshot" classname="${key}" time="1"><failure type="AssertionError" message="expected snapshot"/></testcase>
|
||||
</testsuite></testsuites>`);
|
||||
fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), ledger.map(row => JSON.stringify(row)).join('\n'));
|
||||
const manifest: any = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, profile: 'full', selection: { e2e: null, judges: null }, entries: [{ file: key, slice: 1, status: 'planned', estimatedMs: 1 }] };
|
||||
const result: any = { version: 1, tier: 'gate', profile: 'full', selection: { e2e: null, judges: null }, sliceIndex: 1, sliceCount: 1, attempt: 1,
|
||||
outcomes: [{ files: [key], status: 'failed', exitCode: 1, elapsedMs: 10, executedTests: 2, skippedTests: 0 }] };
|
||||
try { return junitCensus(manifest, [{ root, result }], new Map([[key, records]]), {}); } finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
};
|
||||
const labelled = [
|
||||
{ name: 'browse basic commands', passed: true, cost_usd: 0.05, cost_case_id: 'browse-basic' },
|
||||
{ name: 'browse snapshot flags', passed: false, cost_usd: 0.06, cost_case_id: 'browse-snapshot', exit_reason: 'timeout', failure_class: 'infra', error: 'record error' },
|
||||
];
|
||||
|
||||
test('display-label records bind their cost by cost_case_id in a multi-case shard', () => {
|
||||
const cases = census(labelled).ruleCases;
|
||||
expect(cases.map(c => [c.id, c.costUsd, c.costKnown])).toEqual([['browse-basic', 0.05, true], ['browse-snapshot', 0.06, true]]);
|
||||
});
|
||||
|
||||
test('the failed record and failure class stay what the named records give, as without cost_case_id', () => {
|
||||
const bound = census(labelled).ruleCases.find(c => c.id === 'browse-snapshot')!;
|
||||
const unbound = census(labelled.map(({ cost_case_id, ...rest }) => rest)).ruleCases.find(c => c.id === 'browse-snapshot')!;
|
||||
expect(unbound).toMatchObject({ costUsd: 0, costKnown: false });
|
||||
expect({ failureClass: bound.failureClass, error: bound.error, line: bound.line?.replace(/ {2}\[slice.*$/, '') })
|
||||
.toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') });
|
||||
expect(bound.failureClass).toBe('assertion');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user