mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
fix: the headline adds only streamed session cost; cache writes without a 5m key
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
8364871096
commit
8b572eefec
5 files changed
+17
-10
No files matched your search
+2
-1
@@ -166,7 +166,8 @@ export function anthropicUsageCostUsd(usage: AnthropicUsage, model: string | und
|
||||
if (!row) return undefined;
|
||||
const count = (value: unknown) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
|
||||
const oneHour = count(usage.cache_creation?.ephemeral_1h_input_tokens);
|
||||
const fiveMinute = usage.cache_creation ? count(usage.cache_creation.ephemeral_5m_input_tokens) : count(usage.cache_creation_input_tokens);
|
||||
const fiveMinute = typeof usage.cache_creation?.ephemeral_5m_input_tokens === 'number'
|
||||
? count(usage.cache_creation.ephemeral_5m_input_tokens) : Math.max(0, count(usage.cache_creation_input_tokens) - oneHour);
|
||||
const cachedRate = row.cached_input_per_mtok ?? row.input_per_mtok * 0.1;
|
||||
return (count(usage.input_tokens) * row.input_per_mtok
|
||||
+ count(usage.cache_read_input_tokens) * cachedRate
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"a4ef799c233c65e9209a4b654360476fdbf51802","since":6,"nonBehavioral":"the trial model comes from the named records only, as before cost_case_id; cost fields only, no verdict, failure class or series key changes"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
|
||||
@@ -230,7 +230,7 @@ export interface JUnitCensus {
|
||||
deselected: Map<string, number>;
|
||||
/** Per shard key: skip reason -> skipped testcase count. */
|
||||
shardSkipReasons: Map<string, Map<string, number>>;
|
||||
/** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */
|
||||
/** Streamed session cost (sessions without a terminal result), which no collector record carries; the headline adds it to the collectors' total_cost_usd. */
|
||||
ledgerCostUsd: number;
|
||||
/** `<shard key> :: <test name> (<outcome>)` for every testcase no rule attributes. */
|
||||
unattributed: string[];
|
||||
@@ -253,17 +253,12 @@ export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-fl
|
||||
*/
|
||||
export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
|
||||
if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
|
||||
const unrecorded = unrecordedCosts(records, sessions);
|
||||
const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
|
||||
const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c));
|
||||
const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded);
|
||||
return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) };
|
||||
}
|
||||
|
||||
/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */
|
||||
function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array<number | undefined> {
|
||||
return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
|
||||
}
|
||||
|
||||
const knownSum = (costs: Array<number | undefined>) => costs.reduce<number>((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0);
|
||||
|
||||
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
|
||||
@@ -311,7 +306,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const cost = caseCost(id, mine, sessions);
|
||||
census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions));
|
||||
census.ledgerCostUsd += knownSum(sessions.map(s => s.billed === false ? s.streamed_cost_usd : 0));
|
||||
const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
|
||||
const failureClass = credentials ? 'infra' : recorded;
|
||||
const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
|
||||
|
||||
@@ -137,6 +137,15 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr
|
||||
expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown');
|
||||
});
|
||||
|
||||
test('a billed session\'s cost is already in its collector total, so the verdict never adds it again (Codex host, unbound label record)', () => {
|
||||
const r = report('periodic', dir => {
|
||||
results(dir, 'skill-e2e-cso', [{ name: '/cso full audit', suite: 'cso', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.5 }]);
|
||||
fs.writeFileSync(path.join(shard(dir, 'skill-e2e-cso'), 'session-ledger.jsonl'),
|
||||
JSON.stringify({ key: 'k', runner: 'codex', started_at: 't', elapsed_ms: 1, end: 'completed', billed: true, cost_usd: 0.5 }) + '\n');
|
||||
});
|
||||
expect(collector(r.dir).verdict.costUsd).toBe(0.5);
|
||||
});
|
||||
|
||||
// Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard
|
||||
// wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class);
|
||||
// fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped;
|
||||
|
||||
@@ -23,6 +23,8 @@ describe('pricing usage', () => {
|
||||
.toBeCloseTo(4 + 20 + 0.2 + 5, 6);
|
||||
expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_5m_input_tokens: 1_000_000, ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5'))
|
||||
.toBeCloseTo(5 + 16, 6);
|
||||
expect(anthropicUsageCostUsd({ cache_creation_input_tokens: 3_000_000, cache_creation: { ephemeral_1h_input_tokens: 2_000_000 } }, 'claude-opus-5-5'))
|
||||
.toBeCloseTo(5 + 16, 6);
|
||||
expect(anthropicUsageCostUsd({ input_tokens: 10 }, 'no-such-model')).toBeUndefined();
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user