diff --git a/test/fixtures/gate-cost/l3-screen-costs.json b/test/fixtures/gate-cost/l3-screen-costs.json deleted file mode 100644 index ef694e92a..000000000 --- a/test/fixtures/gate-cost/l3-screen-costs.json +++ /dev/null @@ -1 +0,0 @@ -{"source": "GSTA-49 l3.json: 92 merged PRs of 2026-10-03..18, each [today's selection cost > 0 ? 1 : 0, screen cost on the 106.0259 full-gate scale]", "rows": [[0, 14.373095972222224], [1, 14.050873750000001], [1, 14.050873750000001], [1, 106.02593819444442], [1, 106.02593819444442], [1, 106.02593819444442], [1, 102.64969972222224], [1, 14.050873750000001], [0, 14.050873750000001], [1, 15.564207083333335], [0, 15.564207083333335], [0, 0], [1, 14.254477750000001], [0, 14.050873750000001], [0, 14.050873750000001], [1, 14.050873750000001], [0, 14.050873750000001], [1, 53.22042332777778], [1, 80.51275179444444], [1, 48.76101488888888], [1, 47.29440655], [1, 46.47003072777778], [0, 15.564207083333335], [1, 86.24345065555555], [1, 14.050873750000001], [1, 50.87805546111111], [1, 27.420873750000002], [1, 57.606871355555555], [1, 106.02593819444442], [1, 14.050873750000001], [1, 106.02593819444442], [1, 14.050873750000001], [1, 14.050873750000001], [1, 14.373095972222224], [1, 76.32757010555557], [1, 106.02593819444442], [1, 14.050873750000001], [1, 16.26434822222222], [0, 14.050873750000001], [1, 92.61747750000002], [1, 78.07442481111111], [1, 14.69976263888889], [1, 31.99004913888889], [1, 14.050873750000001], [1, 14.050873750000001], [1, 19.93114441666667], [1, 59.44910602222222], [1, 14.050873750000001], [1, 106.02593819444442], [1, 22.153795483333337], [1, 14.050873750000001], [1, 78.48570570000001], [1, 62.558475055555554], [1, 106.02593819444442], [1, 14.050873750000001], [1, 50.60545933333333], [1, 79.98434407222223], [1, 79.57949141666667], [1, 67.68989962777778], [0, 0], [1, 106.02593819444442], [1, 14.050873750000001], [1, 14.050873750000001], [1, 14.69976263888889], [1, 95.98650832222224], [1, 14.69976263888889], [0, 15.564207083333335], [1, 20.580033305555556], [0, 15.242285772222223], [1, 106.02593819444442], [1, 33.21087375], [1, 106.02593819444442], [1, 106.02593819444442], [0, 15.564207083333335], [0, 14.050873750000001], [0, 15.564207083333335], [1, 106.02593819444442], [1, 106.02593819444442], [1, 106.02593819444444], [0, 15.564207083333335], [1, 106.02593819444442], [0, 15.564207083333335], [0, 1.5133333333333334], [0, 15.564207083333335], [1, 19.793095972222222], [1, 106.02593819444442], [0, 15.564207083333335], [1, 106.02593819444442], [1, 106.02593819444442], [1, 21.60934955], [1, 106.02593819444442], [1, 106.02593819444442]]} diff --git a/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/1.0.0-HEAD-e2e-demo.json b/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/1.0.0-HEAD-e2e-demo.json deleted file mode 100644 index 16366fdf3..000000000 --- a/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/1.0.0-HEAD-e2e-demo.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "tier": "e2e", - "tests": [ - { - "name": "/case a", - "tier": "e2e", - "duration_ms": 40500, - "cost_usd": 0.253, - "model": "claude-fable-5-1", - "transcript": [ - { - "type": "assistant" - }, - { - "type": "result", - "modelUsage": { - "claude-fable-5-1": { - "inputTokens": 100, - "outputTokens": 2000, - "cacheReadInputTokens": 100000, - "cacheCreationInputTokens": 10000, - "thinkingTokens": 500, - "costUSD": 0.251 - }, - "claude-haiku-4-5-20251001": { - "inputTokens": 1000, - "outputTokens": 100, - "cacheReadInputTokens": 0, - "cacheCreationInputTokens": 0, - "thinkingTokens": 0, - "costUSD": 0.0015 - } - } - }, - { - "type": "result", - "modelUsage": { - "claude-fable-5-1": { - "inputTokens": 50, - "outputTokens": 0, - "cacheReadInputTokens": 0, - "cacheCreationInputTokens": 0, - "thinkingTokens": 0, - "costUSD": 0.0005 - } - } - } - ] - } - ] -} diff --git a/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/_partial-e2e.json b/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/_partial-e2e.json deleted file mode 100644 index 9c9aba6b4..000000000 --- a/test/fixtures/gate-cost/run-1/paid-slice-1-a1/shards/skill-e2e-demo--case-a/_partial-e2e.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "tests": [ - { - "name": "/case a", - "duration_ms": 40500, - "cost_usd": 99 - } - ] -} diff --git a/test/fixtures/gate-cost/run-1/paid-slice-2-a1/shards/skill-llm-eval/1.0.0-HEAD-llm-judge.json b/test/fixtures/gate-cost/run-1/paid-slice-2-a1/shards/skill-llm-eval/1.0.0-HEAD-llm-judge.json deleted file mode 100644 index b44e69be2..000000000 --- a/test/fixtures/gate-cost/run-1/paid-slice-2-a1/shards/skill-llm-eval/1.0.0-HEAD-llm-judge.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "tier": "llm-judge", - "tests": [ - { - "name": "demo/SKILL.md judge", - "tier": "llm-judge", - "duration_ms": 20000, - "cost_usd": 0.06 - } - ] -} diff --git a/test/fixtures/gate-cost/run-1/trial-outcomes-pr-a1/trial-outcomes.jsonl b/test/fixtures/gate-cost/run-1/trial-outcomes-pr-a1/trial-outcomes.jsonl deleted file mode 100644 index 4f1073561..000000000 --- a/test/fixtures/gate-cost/run-1/trial-outcomes-pr-a1/trial-outcomes.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"schema": "gstack-trial-outcome/v1", "tier": "gate", "attempt": 1, "run_id": "1", "trial": 1, "outcome": "passed", "case": "case-a", "file": "test/skill-e2e-demo.test.ts", "kind": "rule", "duration_ms": 40000, "cost_usd": 0, "cost_known": false, "sessions": [{"runner": "claude-p"}]} -{"schema": "gstack-trial-outcome/v1", "tier": "gate", "attempt": 1, "run_id": "1", "trial": 1, "outcome": "passed", "case": "case-pty", "file": "test/skill-e2e-pty-demo.test.ts", "kind": "rule", "duration_ms": 100000, "cost_usd": 0, "cost_known": false, "sessions": [{"runner": "pty"}]} -{"schema": "gstack-trial-outcome/v1", "tier": "gate", "attempt": 1, "run_id": "1", "trial": 1, "outcome": "passed", "case": "demo/SKILL.md judge", "file": "test/skill-llm-eval.test.ts", "kind": "judge", "duration_ms": 20000, "cost_usd": 0.06} diff --git a/test/gate-cost.test.ts b/test/gate-cost.test.ts index e51d25f5b..3d5092b4a 100644 --- a/test/gate-cost.test.ts +++ b/test/gate-cost.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, test } from 'bun:test'; +import { afterAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; +import * as os from 'node:os'; import * as path from 'node:path'; import { PyRandom, SCREEN_LABEL, caseCostTable, changedLinesByFile, gateCostBreakdown, heuristicPreambleOnly, parseEvalReport, @@ -7,8 +8,53 @@ import { } from '../lib/gate-cost'; import { buildReport, checkoutSelectionDeps, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type ReplayReport } from '../scripts/gate-cost-replay'; -const FIXTURES = path.join(import.meta.dir, 'fixtures', 'gate-cost'); -const run1 = () => readGateRecord(path.join(FIXTURES, 'run-1'), '1'); +/** + * A one-run artifact tree, written to a temporary directory: tracked files + * under test/fixtures would need a FREE_FIXTURES entry, and that registry is + * a paid-runner input that selects every PR-profile case. + */ +const RUN_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-cost-run-')); +afterAll(() => fs.rmSync(RUN_DIR, { recursive: true, force: true })); +const usage = (inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number, thinkingTokens: number, costUSD: number) => + ({ inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens, thinkingTokens, costUSD }); +const FIXTURE_FILES: Record = { + 'paid-slice-1-a1/shards/skill-e2e-demo--case-a/1.0.0-HEAD-e2e-demo.json': { tests: [{ + name: '/case a', tier: 'e2e', duration_ms: 40_500, cost_usd: 0.253, model: 'claude-fable-5-1', transcript: [ + { type: 'assistant' }, + { type: 'result', modelUsage: { 'claude-fable-5-1': usage(100, 2000, 100_000, 10_000, 500, 0.251), 'claude-haiku-4-5-20251001': usage(1000, 100, 0, 0, 0, 0.0015) } }, + { type: 'result', modelUsage: { 'claude-fable-5-1': usage(50, 0, 0, 0, 0, 0.0005) } }, + ] }] }, + 'paid-slice-1-a1/shards/skill-e2e-demo--case-a/_partial-e2e.json': { tests: [{ name: '/case a', duration_ms: 40_500, cost_usd: 99 }] }, + 'paid-slice-2-a1/shards/skill-llm-eval/1.0.0-HEAD-llm-judge.json': { tests: [{ name: 'demo/SKILL.md judge', tier: 'llm-judge', duration_ms: 20_000, cost_usd: 0.06 }] }, +}; +for (const [file, body] of Object.entries(FIXTURE_FILES)) { + fs.mkdirSync(path.dirname(path.join(RUN_DIR, file)), { recursive: true }); + fs.writeFileSync(path.join(RUN_DIR, file), JSON.stringify(body)); +} +const trial = (caseId: string, file: string, kind: string, durationMs: number, runner?: string) => + JSON.stringify({ schema: 'gstack-trial-outcome/v1', case: caseId, file, kind, duration_ms: durationMs, cost_usd: 0, ...(runner ? { sessions: [{ runner }] } : {}) }); +fs.mkdirSync(path.join(RUN_DIR, 'trial-outcomes-pr-a1')); +fs.writeFileSync(path.join(RUN_DIR, 'trial-outcomes-pr-a1', 'trial-outcomes.jsonl'), [ + trial('case-a', 'test/skill-e2e-demo.test.ts', 'rule', 40_000, 'claude-p'), + trial('case-pty', 'test/skill-e2e-pty-demo.test.ts', 'rule', 100_000, 'pty'), + 'not json', + trial('demo/SKILL.md judge', 'test/skill-llm-eval.test.ts', 'judge', 20_000), +].join('\n') + '\n'); +const run1 = () => readGateRecord(RUN_DIR, '1'); + +/** GSTA-49's l3.json: 92 merged PRs of 2026-10-03..18, [today's selection cost > 0 ? 1 : 0, screen cost on the 106.0259 full-gate scale]. */ +const L3_ROWS: Array<[number, number]> = [ + [0,14.3731], [1,14.0509], [1,14.0509], [1,106.0259], [1,106.0259], [1,106.0259], [1,102.6497], [1,14.0509], [0,14.0509], [1,15.5642], + [0,15.5642], [0,0], [1,14.2545], [0,14.0509], [0,14.0509], [1,14.0509], [0,14.0509], [1,53.2204], [1,80.5128], [1,48.761], + [1,47.2944], [1,46.47], [0,15.5642], [1,86.2435], [1,14.0509], [1,50.8781], [1,27.4209], [1,57.6069], [1,106.0259], [1,14.0509], + [1,106.0259], [1,14.0509], [1,14.0509], [1,14.3731], [1,76.3276], [1,106.0259], [1,14.0509], [1,16.2643], [0,14.0509], [1,92.6175], + [1,78.0744], [1,14.6998], [1,31.99], [1,14.0509], [1,14.0509], [1,19.9311], [1,59.4491], [1,14.0509], [1,106.0259], [1,22.1538], + [1,14.0509], [1,78.4857], [1,62.5585], [1,106.0259], [1,14.0509], [1,50.6055], [1,79.9843], [1,79.5795], [1,67.6899], [0,0], + [1,106.0259], [1,14.0509], [1,14.0509], [1,14.6998], [1,95.9865], [1,14.6998], [0,15.5642], [1,20.58], [0,15.2423], [1,106.0259], + [1,33.2109], [1,106.0259], [1,106.0259], [0,15.5642], [0,14.0509], [0,15.5642], [1,106.0259], [1,106.0259], [1,106.0259], [0,15.5642], + [1,106.0259], [0,15.5642], [0,1.5133], [0,15.5642], [1,19.7931], [1,106.0259], [0,15.5642], [1,106.0259], [1,106.0259], [1,21.6093], + [1,106.0259], [1,106.0259], +]; describe('paid report artifacts', () => { test('reads eval reports and trial outcomes, skipping partial reports', () => { @@ -120,8 +166,7 @@ describe('merge-train model', () => { expect(new PyRandom(2 ** 40 + 7).random()).toBe(0.6137037779936511); }); - const l3 = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'l3-screen-costs.json'), 'utf8')).rows as Array<[number, number]>; - const prs = l3.map(([relevant, cost]) => ({ paidRelevant: relevant === 1, screenUsd: cost * 93.14 / 106.0259 })); + const prs = L3_ROWS.map(([relevant, cost]) => ({ paidRelevant: relevant === 1, screenUsd: cost * 93.14 / 106.0259 })); const model = (policy: Parameters[0]['policy']) => simulateTrains({ prs, policy, windowDays: 15.1, trainSize: 5, outsideRed: 0.15 }); test('reproduces the measurement sim (trains of 5, o=0.15, 6%/25% red, 400 reps, seed 1)', () => {