mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 14:34:42 +01:00
fix: the paid report headline counts ledger-only session cost once
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
96250a59e0
commit
fa5629423d
3 files changed
+30
-6
No files matched your search
@@ -13,7 +13,7 @@
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"b00d662e502ebab7aa7050013dbb2ddd6457baf8","since":6,"nonBehavioral":"caseCost adds streamed cost of record-less sessions to record sums and honors record cost_known; cost fields only, no verdict or failure class changes"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"6adbde198309328bae17c3b17d199963e19a4c7e","since":6,"nonBehavioral":"the paid report headline adds session cost no collector record carries; cost fields only, no verdict or failure class changes"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
|
||||
@@ -230,6 +230,8 @@ export interface JUnitCensus {
|
||||
deselected: Map<string, number>;
|
||||
/** Per shard key: skip reason -> skipped testcase count. */
|
||||
shardSkipReasons: Map<string, Map<string, number>>;
|
||||
/** Session cost no collector record carries; the headline adds it to the collectors' total_cost_usd. */
|
||||
ledgerCostUsd: number;
|
||||
/** `<shard key> :: <test name> (<outcome>)` for every testcase no rule attributes. */
|
||||
unattributed: string[];
|
||||
}
|
||||
@@ -251,13 +253,19 @@ export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-fl
|
||||
*/
|
||||
export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
|
||||
if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
|
||||
const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
|
||||
const unrecorded = unrecordedCosts(records, sessions);
|
||||
const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c));
|
||||
const total = [...records.map(r => Number(r?.cost_usd) || 0), ...unrecorded.filter((c): c is number => typeof c === 'number' && Number.isFinite(c))]
|
||||
.reduce((sum, c) => sum + c, 0);
|
||||
const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded);
|
||||
return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) };
|
||||
}
|
||||
|
||||
/** Session costs no collector record carries (so no collector total_cost_usd counts them): streamed usage, and a billed session's own cost when the trial has no record. */
|
||||
function unrecordedCosts(records: any[], sessions: SessionLedgerRow[]): Array<number | undefined> {
|
||||
return sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
|
||||
}
|
||||
|
||||
const knownSum = (costs: Array<number | undefined>) => costs.reduce<number>((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0);
|
||||
|
||||
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
|
||||
/** The red-census guide the report and every eval:pass-rates view link. */
|
||||
export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
|
||||
@@ -269,7 +277,8 @@ export function sliceArtifactName(root: string, slice: number, attempt: number):
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [],
|
||||
ledgerCostUsd: 0 };
|
||||
const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
|
||||
for (const { root, result } of artifacts) {
|
||||
for (const outcome of result.outcomes) {
|
||||
@@ -302,6 +311,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const cost = caseCost(id, mine, sessions);
|
||||
census.ledgerCostUsd += knownSum(unrecordedCosts(mine, sessions));
|
||||
const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
|
||||
const failureClass = credentials ? 'infra' : recorded;
|
||||
const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
|
||||
@@ -796,7 +806,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
...(spend.executions ? { reruns: spend } : {}),
|
||||
actionRequired: verdict.problems.length,
|
||||
wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null,
|
||||
costUsd: Math.round(costUsd * 100) / 100,
|
||||
costUsd: Math.round((costUsd + census.ledgerCostUsd) * 100) / 100,
|
||||
redispatchEligible: red && infraOnly(verdict.problems),
|
||||
};
|
||||
const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
|
||||
|
||||
@@ -123,6 +123,20 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr
|
||||
expect(collector(r.dir).verdict.costUsd).toBe(0.75);
|
||||
});
|
||||
|
||||
test('a record-less trial\'s streamed session cost reaches the trial and the verdict total once (GSTA-150 outside voice: $0.65 known, headline $0.00)', () => {
|
||||
const ledger = (dir: string, slug: string, costs: number[]) => fs.writeFileSync(path.join(shard(dir, slug), 'session-ledger.jsonl'),
|
||||
costs.map(cost => JSON.stringify({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', billed: false, streamed_cost_usd: cost })).join('\n') + '\n');
|
||||
const r = report('periodic', dir => {
|
||||
ledger(dir, 'skill-e2e-qa-workflow--qa-fix-loop', [0.4, 0.25]);
|
||||
results(dir, 'skill-e2e-benchmark-providers', [{ name: 'benchmark-providers-live', suite: 's', tier: 'e2e', passed: true, duration_ms: 1, cost_usd: 0.1 }]);
|
||||
ledger(dir, 'skill-e2e-benchmark-providers', [0.5]);
|
||||
});
|
||||
expect(r.history.find(record => record.case === 'qa-fix-loop')).toMatchObject({ outcome: 'passed', cost_usd: 0.65 });
|
||||
expect(r.history.find(record => record.case === 'benchmark-providers-live')).toMatchObject({ cost_usd: 0.6 });
|
||||
expect(collector(r.dir).verdict.costUsd).toBe(1.25);
|
||||
expect(r.out).toContain('cost $1.25 known + 5 of 7 trial(s) cost unknown');
|
||||
});
|
||||
|
||||
// Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard
|
||||
// wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class);
|
||||
// fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped;
|
||||
|
||||
Reference in new issue
Block a user