diff --git a/CHANGELOG.md b/CHANGELOG.md index f9839d4ef..a080bff1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.91.137.0] - 2026-10-18 + +**`scripts/gate-cost-replay.ts` measures what the paid PR gate costs, and what each selection or merge-train policy would have cost, from recorded artifacts and for free.** On the nine full gates of Oct 17 and 18 it reproduces GSTA-49's measurement: $93.14 recorded per gate, 97.1% of it Fable 5.1 (the model under test), 49.8% cache writes and 40.3% output. Over the 92 squash merges of Oct 3 to 18, today's selector averages $78.67 per merged PR on the $106.19 full-gate scale, R4's exercised-map shadow $64.27, and the screening rule $45.17 (screen-provisional). This is lane T0 of the plan to cut the paid gate bill about 4x; nothing changes what any gate runs. + +### Added +- **`bun run scripts/gate-cost-replay.ts --runs [--since/--until | --prs] [--json]`.** It downloads the runs' `paid-slice-*` and `trial-outcomes` artifacts read-only with `gh` into `.context/gate-cost-cache` once, then prints recorded $ per gate by model and by token class (priced through `lib/pricing.ts`), a per-case mean cost table, and each merged PR's selection cost under today's selector, R4's shadow and the screening rule. A case whose runs record no cost (the PTY sessions) is estimated from its duration and marked estimated. +- **Screening figures print as `screen-provisional`.** The rule's preamble detector is a line-commonality heuristic, not T2's hunk classifier, so the figures stay heuristic until T2 lands. A `package.json` change that only moves the version is neither selected nor a canary trigger, and all 24 judges always count. +- **A seeded merge-train model.** Every merged PR lands in a train. PRs carry defects at the screened (6%) or unscreened (25%) rate, gates flake at 10% or 15%, each red pays a $6 bisect, a culprit is dropped and requeued, and an unattributed red gets one rebuild. It reports $ per week and per merged PR against $9,950 and $8,400 a week, plus screen spend, trains, gate attempts, bisects, rebuilds, stalls and red shares, for screening every PR, 80%, half, the cheapest or the most expensive half, or none. It uses CPython's Mersenne Twister, so seed 1 reproduces the measurement's sim.py exactly (trains of 5, 15% outside reds: $3,403, $2,672 and $1,986 a week for all, half and none). +- Logic lives in `lib/gate-cost.ts`; the command is documented in docs/TESTING_INTERNALS.md#replay-gate-cost and CONTRIBUTING's "Which command do I run?". + +### If you… +- **…want current numbers:** run `bun run scripts/gate-cost-replay.ts --runs --since --until `. A shallow checkout tells you which `git fetch --shallow-since` to run first. + ## [1.91.136.0] - 2026-10-18 **/cso release qualification counts a tested claim as false only when trusted evidence could have certified it.** In the second paid runtime qualification, the falsely-tested gate counted 33 claims. 12 of them were on the evaluator's own fixture instrumentation and 2 were on real incidental issues, which only the seeded repair verification could ever certify. Garry chose option 1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0486eee65..31499643f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -205,6 +205,7 @@ the new defaults. | See a case's pass rate across recent weekly runs | `bun run eval:pass-rates --case ` | Free | `gh` with repo read access | | Measure one red case alone against the ship bar (MEETS 9/10) | `bun run scripts/ship-measure.ts measure --case --round baseline` ([bar](docs/TESTING_INTERNALS.md#ship-measure)) | API spend for 10 trials (12 for behavior); asks above $2/trial, stops at $25/case | Same as paid coverage | | Measure main's flakiest gate cases (the weekly sweep) | `gh workflow run eval-sweep.yml --ref main [-f k=5] [-f cap_usd=150] [-f dry_run=true]`, or locally `bun run scripts/ship-measure.ts sweep --dry-run` ([sweep](docs/TESTING_INTERNALS.md#ship-measure-sweep)) | API spend up to $150 per 7 days (`ship_measure_sweep_budget_usd`); `--dry-run` is free | `gh` with repo read access; CI dispatch needs workflow rights | +| Replay what the paid gate costs and what each selection or train policy would have cost ([replay](docs/TESTING_INTERNALS.md#replay-gate-cost)) | `bun run scripts/gate-cost-replay.ts --runs --since --until ` (screening figures print as screen-provisional, heuristic until T2 lands) | Free; `gh` reads, cached in `.context/gate-cost-cache` | `gh` with repo read access | | See the audit success metrics and weekly health | `bun run test:health [--since-days 7] [--json] [--enforce]` | Free; 5–20 minutes of `gh` reads | `gh` with repo read access | Old command names are listed under [Retired commands](#retired-commands). diff --git a/VERSION b/VERSION index ebbca0fda..bc8cb22f2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.136.0 +1.91.137.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index e5bcd63fa..c959a41e8 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.136.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.137.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index c3d8aafb9..9f2d95941 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1199,6 +1199,39 @@ PR labels: `evals-no-debounce` sets the quiet window to 0 (Free Tests is still required) and `evals-fresh` turns receipt reuse and verdict reuse off. Labels apply from the next push, because adding a label does not start `evals.yml`. +### Replay gate cost + +`bun run scripts/gate-cost-replay.ts --runs [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202] [--json]` +measures the paid PR gate's bill from recorded artifacts, for free. `--runs` +names complete full-gate runs; their `paid-slice-*` and `trial-outcomes` +artifacts are downloaded read-only with `gh` into `.context/gate-cost-cache` +once and reused from there. It prints: + +- recorded $ per gate (mean, min, max), split by model and by token class + (cache write, output including thinking, cache read, uncached input), priced + through `lib/pricing.ts`; +- a per-case mean cost table (`--cases` prints all of it); a case whose runs + record no cost (PTY sessions) is estimated from its duration and marked + estimated, at recorded dollars per recorded session-second unless + `--usd-per-second` pins the rate; +- for each squash merge on origin/main's first-parent history in the window + (bare dates are UTC days, `--until` inclusive), the selection cost under + today's selector, R4's exercised-map shadow and the screening rule, and the + mean, median and mean case count per merged PR over every merged PR in the + window, zeros included; +- a seeded merge-train model (every merged PR lands in a train; screened and + unscreened defect rates, outside reds, a $6 bisect per red, culprits + requeued, one rebuild per unattributed red) for each screening policy, with + $ per week and per merged PR and the multiple against $9,950 and $8,400 a week. + +Screening figures are labelled `screen-provisional`: the rule's preamble +detector is a line-commonality heuristic, not T2's hunk classifier, so they +stay heuristic until T2 lands. A `package.json` change that only moves the +version is neither selected nor a canary trigger, and all 24 judges always run. +The selectors are this checkout's, so an old merge is replayed against today's +maps. A shallow checkout says which `git fetch --shallow-since` to run. Logic: +`lib/gate-cost.ts`; tests: `test/gate-cost.test.ts`. + ## Free suite runner, judge reuse and engine skips Moved verbatim from CLAUDE.md (#2096 size limit). diff --git a/lib/gate-cost.ts b/lib/gate-cost.ts new file mode 100644 index 000000000..53b88e5c4 --- /dev/null +++ b/lib/gate-cost.ts @@ -0,0 +1,610 @@ +/** + * gate-cost — measurement-as-code for the paid PR gate's bill (GSTA-49 plan + * lane T0; numbers in its MEASUREMENT.md). Pure logic: callers pass artifact + * contents, changed-file lists and selector functions in; nothing here reads + * files, runs git or calls GitHub. scripts/gate-cost-replay.ts is the CLI. + * + * Four parts: + * 1. Paid report artifacts -> per-case records, a gate's recorded cost by + * model and by token class (priced through lib/pricing.ts). + * 2. A per-case mean cost table across several gates; a case with no + * recorded cost is estimated from its duration and marked estimated. + * 3. Selection cost of one merged PR under today's selector, R4's shadow + * selection and the screening rule. The screening rule is labelled + * `screen-provisional` wherever it is printed: its preamble detector is + * a line-commonality heuristic, not T2's production hunk classifier. + * 4. A seeded Monte Carlo of merge trains (every merged PR lands in a train; + * defects, outside flakes, bisects, requeues, one rebuild per + * unattributed red). It ports the measurement's sim.py and uses CPython's + * Mersenne Twister, so the same seed gives the same figures. + */ +import { pricingRow } from './pricing'; + +export const SCREEN_LABEL = 'screen-provisional'; +/** Anthropic bills 5-minute prompt-cache writes at 1.25x the model's input price (lib/pricing.ts excludes cache writes). */ +export const CACHE_WRITE_INPUT_MULTIPLIER = 1.25; +export const TODAY_WEEKLY_USD = 9_950; +export const AFTER_GSTA48_WEEKLY_USD = 8_400; +/** Files that add the screening canary set (T2's full-gate list); package.json only when more than its version changed. */ +export const SCREEN_FULL_GATE_FILES = [ + 'setup', 'bun.lock', '.github/docker/Dockerfile.ci', '.github/workflows/evals.yml', + 'scripts/build.sh', 'scripts/gen-skill-docs.ts', 'package.json', +] as const; +/** The heuristic preamble detector runs only on diffs that change at least this many SKILL.md files. */ +export const PREAMBLE_MIN_SKILL_FILES = 10; +/** A SKILL.md change counts as preamble-only when every changed line also changed in this share of the changed SKILL.md files. */ +export const PREAMBLE_LINE_SHARE = 0.6; + +// --- 1. Artifacts --- + +export interface UsageTotals { + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + cacheCreationInputTokens: number; + thinkingTokens: number; + costUSD: number; +} + +export interface EvalTestRecord { + shard: string; + file: string | null; + name: string; + tier: string | null; + model: string | null; + costUsd: number; + durationMs: number; + /** Summed over every `result` event in the test's transcript. */ + modelUsage: Record; +} + +export interface TrialRecord { + caseId: string; + file: string; + kind: string; + durationMs: number; + runner: string | null; +} + +export interface GateRecord { + runId: string; + tests: EvalTestRecord[]; + trials: TrialRecord[]; +} + +const USAGE_KEYS = ['inputTokens', 'outputTokens', 'cacheReadInputTokens', 'cacheCreationInputTokens', 'thinkingTokens', 'costUSD'] as const; +const emptyUsage = (): UsageTotals => ({ inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, thinkingTokens: 0, costUSD: 0 }); +const num = (value: unknown): number => (typeof value === 'number' && Number.isFinite(value) ? value : 0); + +/** `skill-e2e-deploy--canary-workflow.t2` -> `test/skill-e2e-deploy.test.ts` (scripts/lib/paid-select.ts shardSlug, reversed). */ +export function shardTestFile(shard: string): string | null { + const base = shard.replace(/\.t\d+$/, '').split('--')[0]; + return base ? `test/${base}.test.ts` : null; +} + +/** One eval JSON from a paid-slice shard directory; anything without a `tests` array yields no records. */ +export function parseEvalReport(shard: string, report: unknown): EvalTestRecord[] { + const tests = (report as { tests?: unknown } | null)?.tests; + if (!Array.isArray(tests)) return []; + return tests.filter((test): test is Record => !!test && typeof test === 'object').map(test => { + const modelUsage: Record = {}; + for (const event of Array.isArray(test.transcript) ? test.transcript : []) { + if (event?.type !== 'result' || !event.modelUsage || typeof event.modelUsage !== 'object') continue; + for (const [model, usage] of Object.entries(event.modelUsage as Record>)) { + const totals = modelUsage[model] ??= emptyUsage(); + for (const key of USAGE_KEYS) totals[key] += num(usage?.[key]); + } + } + return { + shard, file: shardTestFile(shard), name: String(test.name ?? ''), + tier: typeof test.tier === 'string' ? test.tier : null, model: typeof test.model === 'string' ? test.model : null, + costUsd: num(test.cost_usd), durationMs: num(test.duration_ms), modelUsage, + }; + }); +} + +/** trial-outcomes.jsonl; a torn or foreign line is skipped. */ +export function parseTrialOutcomes(text: string): TrialRecord[] { + const trials: TrialRecord[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + let row: Record; + try { row = JSON.parse(line); } catch { continue; } + if (typeof row?.case !== 'string' || typeof row.file !== 'string') continue; + const sessions = Array.isArray(row.sessions) ? row.sessions as Array<{ runner?: unknown }> : []; + trials.push({ + caseId: row.case, file: row.file, kind: typeof row.kind === 'string' ? row.kind : 'rule', + durationMs: num(row.duration_ms), runner: typeof sessions[0]?.runner === 'string' ? sessions[0].runner : null, + }); + } + return trials; +} + +export interface TokenClass { tokens: number; usd: number } +export interface ModelTokenClasses { + cacheWrite: TokenClass; + output: TokenClass & { thinkingTokens: number }; + cacheRead: TokenClass; + uncachedInput: TokenClass; + pricedUsd: number; + recordedUsd: number; + priced: boolean; +} + +export interface GateCostBreakdown { + recordedUsd: number; + /** modelUsage costUSD per model; a test's cost that modelUsage does not cover goes to its row model (judges: `llm-judge`). */ + byModel: Record; + tokenClasses: Record; +} + +/** Price one model's tokens with lib/pricing.ts; output tokens include thinking tokens. */ +export function priceTokenClasses(model: string, usage: UsageTotals): ModelTokenClasses { + const row = pricingRow(model); + const input = row?.input_per_mtok ?? 0; + const cls = (tokens: number, perMtok: number): TokenClass => ({ tokens, usd: tokens * perMtok / 1_000_000 }); + const cacheWrite = cls(usage.cacheCreationInputTokens, input * CACHE_WRITE_INPUT_MULTIPLIER); + const output = { ...cls(usage.outputTokens, row?.output_per_mtok ?? 0), thinkingTokens: usage.thinkingTokens }; + const cacheRead = cls(usage.cacheReadInputTokens, row?.cached_input_per_mtok ?? input * 0.1); + const uncachedInput = cls(usage.inputTokens, input); + return { cacheWrite, output, cacheRead, uncachedInput, pricedUsd: cacheWrite.usd + output.usd + cacheRead.usd + uncachedInput.usd, + recordedUsd: usage.costUSD, priced: row !== undefined }; +} + +export function gateCostBreakdown(gate: Pick): GateCostBreakdown { + const byModel: Record = {}; + const usageByModel: Record = {}; + let recordedUsd = 0; + for (const test of gate.tests) { + recordedUsd += test.costUsd; + let covered = 0; + for (const [model, usage] of Object.entries(test.modelUsage)) { + byModel[model] = (byModel[model] ?? 0) + usage.costUSD; + covered += usage.costUSD; + const totals = usageByModel[model] ??= emptyUsage(); + for (const key of USAGE_KEYS) totals[key] += usage[key]; + } + const rest = test.costUsd - covered; + if (Math.abs(rest) < 0.005) continue; + const label = test.tier === 'llm-judge' ? 'llm-judge' : `${test.model ?? 'model not recorded'} (no modelUsage)`; + byModel[label] = (byModel[label] ?? 0) + rest; + } + const tokenClasses = Object.fromEntries(Object.entries(usageByModel).map(([model, usage]) => [model, priceTokenClasses(model, usage)])); + return { recordedUsd, byModel, tokenClasses }; +} + +export interface GateSummary { + gates: number; + recordedUsd: { mean: number; min: number; max: number }; + byModel: Record; + tokenClasses: Record; +} + +/** Mean over gates of each breakdown figure. */ +export function summarizeGates(gates: Array>): GateSummary { + const breakdowns = gates.map(gateCostBreakdown); + const n = Math.max(1, breakdowns.length); + const totals = breakdowns.map(b => b.recordedUsd); + const byModel: Record = {}; + const tokenClasses: Record = {}; + for (const b of breakdowns) { + for (const [model, usd] of Object.entries(b.byModel)) byModel[model] = (byModel[model] ?? 0) + usd / n; + for (const [model, c] of Object.entries(b.tokenClasses)) { + const t = tokenClasses[model] ??= { + cacheWrite: { tokens: 0, usd: 0 }, output: { tokens: 0, usd: 0, thinkingTokens: 0 }, cacheRead: { tokens: 0, usd: 0 }, + uncachedInput: { tokens: 0, usd: 0 }, pricedUsd: 0, recordedUsd: 0, priced: c.priced, + }; + for (const key of ['cacheWrite', 'output', 'cacheRead', 'uncachedInput'] as const) { + t[key].tokens += c[key].tokens / n; + t[key].usd += c[key].usd / n; + } + t.output.thinkingTokens += c.output.thinkingTokens / n; + t.pricedUsd += c.pricedUsd / n; + t.recordedUsd += c.recordedUsd / n; + } + } + return { + gates: breakdowns.length, + recordedUsd: totals.length ? { mean: totals.reduce((a, b) => a + b, 0) / n, min: Math.min(...totals), max: Math.max(...totals) } : { mean: 0, min: 0, max: 0 }, + byModel, tokenClasses, + }; +} + +// --- 2. Per-case cost table --- + +/** + * Bind each trial outcome to the eval test entry that recorded its cost: an + * unused entry named like the case first, else the unused entry of the same + * test file nearest in duration (within 5 s or 10%). `null` = no recorded + * cost (PTY sessions and the few runners that do not report one). + */ +export function recordedCaseCosts(gate: GateRecord): Array<{ trial: TrialRecord; costUsd: number | null }> { + const used = new Set(); + return gate.trials.map(trial => { + const unused = gate.tests.map((test, index) => ({ test, index })).filter(({ index }) => !used.has(index)); + const named = unused.filter(({ test }) => test.name === trial.caseId); + const pool = named.length ? named : unused.filter(({ test }) => test.file === trial.file); + const best = pool.sort((a, b) => Math.abs(a.test.durationMs - trial.durationMs) - Math.abs(b.test.durationMs - trial.durationMs))[0]; + if (!best || (!named.length && Math.abs(best.test.durationMs - trial.durationMs) >= Math.max(5_000, 0.1 * trial.durationMs))) { + return { trial, costUsd: null }; + } + used.add(best.index); + return { trial, costUsd: best.test.costUsd }; + }); +} + +export interface CaseCostEntry { + caseId: string; + file: string; + kind: string; + runner: string | null; + costUsd: number; + estimated: boolean; + /** Gates that recorded a cost for the case (0 when estimated). */ + recordedGates: number; + meanDurationMs: number; +} + +export interface CaseCostTable { + cases: Record; + /** $ per session-second used for estimates: recorded cost over the duration of the trials that recorded a nonzero cost. */ + usdPerSecond: number; +} + +export function caseCostTable(gates: GateRecord[], options: { usdPerSecond?: number } = {}): CaseCostTable { + const seen = new Map(); + let recordedUsd = 0, recordedMs = 0; + for (const gate of gates) { + for (const { trial, costUsd } of recordedCaseCosts(gate)) { + const entry = seen.get(trial.caseId) ?? { trial, costs: [], durations: [] }; + seen.set(trial.caseId, entry); + entry.durations.push(trial.durationMs); + if (costUsd === null) continue; + entry.costs.push(costUsd); + if (costUsd > 0) { recordedUsd += costUsd; recordedMs += trial.durationMs; } + } + } + const usdPerSecond = options.usdPerSecond ?? (recordedMs ? recordedUsd / (recordedMs / 1000) : 0); + const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / Math.max(1, values.length); + const cases: Record = {}; + for (const [caseId, { trial, costs, durations }] of seen) { + const meanDurationMs = mean(durations); + cases[caseId] = { + caseId, file: trial.file, kind: trial.kind, runner: trial.runner, meanDurationMs, recordedGates: costs.length, + estimated: costs.length === 0, costUsd: costs.length ? mean(costs) : meanDurationMs / 1000 * usdPerSecond, + }; + } + return { cases, usdPerSecond }; +} + +// --- 3. Selection cost of a merged PR --- + +export interface PrDiff { + pr: number; + sha: string; + files: string[]; + /** package.json differs from the parent only in `version` (scripts/test-pr-profile.ts packageChangeOnlyVersion). */ + packageVersionOnly: boolean; + /** Distinct trimmed changed lines per SKILL.md file; read only when the diff changes PREAMBLE_MIN_SKILL_FILES or more. */ + skillMdChangedLines: Record; +} + +export interface TodaySelection { + e2e: string[]; + judges: string[]; + /** R4's exercised-map shadow (PrProfileSelection.shadow.e2e); null when none was computed. */ + shadowE2e: string[] | null; + mode: string; +} + +export interface SelectionDeps { + gateCases: readonly string[]; + judgeCases: readonly string[]; + /** Today's PR-profile selector (computePaidCaseSelection, profile pr). */ + today(files: string[], packageVersionOnly: boolean): TodaySelection; + /** Declared touchfiles plus harness globals only (profile full); `global` = a harness global file changed. */ + touchfiles(files: string[]): { e2e: string[]; global: boolean }; +} + +export interface SelectorCost { cases: number; judges: number; usd: number } +export interface PrSelectionCost { + pr: number; + sha: string; + files: number; + mode: string; + today: SelectorCost; + shadow: SelectorCost; + screenProvisional: SelectorCost & { canary: boolean; canaryReasons: string[]; preambleOnly: string[] }; +} + +/** + * `git diff -U0` text -> distinct trimmed changed lines per file. Lines that + * start with `++`, `+-`, `-+` or `--` (the file headers among them) are skipped, + * as in the measurement's heuristic. + */ +export function changedLinesByFile(diff: string): Record { + const out: Record> = {}; + let current: string | null = null; + for (const line of diff.split('\n')) { + const header = line.match(/^diff --git a\/.* b\/(.*)$/); + if (header) { current = header[1]!; out[current] ??= new Set(); continue; } + if (current && /^[+-][^+-]/.test(line)) out[current]!.add(line.trim()); + } + return Object.fromEntries(Object.entries(out).map(([file, lines]) => [file, [...lines]])); +} + +/** The line-commonality heuristic standing in for T2's hunk classifier (provisional). */ +export function heuristicPreambleOnly(changed: Record): string[] { + const files = Object.keys(changed); + if (files.length < PREAMBLE_MIN_SKILL_FILES) return []; + const count = new Map(); + for (const lines of Object.values(changed)) for (const line of new Set(lines)) count.set(line, (count.get(line) ?? 0) + 1); + return files.filter(file => changed[file]!.every(line => (count.get(line) ?? 0) >= PREAMBLE_LINE_SHARE * files.length)); +} + +export const isSkillMd = (file: string) => /(^|\/)SKILL\.md$/.test(file); + +function costOf(ids: Iterable, table: CaseCostTable): number { + let usd = 0; + for (const id of ids) usd += table.cases[id]?.costUsd ?? 0; + return usd; +} + +export function fullGateUsd(deps: Pick, table: CaseCostTable): number { + return costOf(deps.gateCases, table) + costOf(deps.judgeCases, table); +} + +/** + * The screen-provisional rule: declared touchfiles and harness globals of the + * files that are not heuristically preamble-only, every judge, and a canary + * set when the diff has preamble-only SKILL.md files, a full-gate file, a + * resolver, or files that reach cases only transitively. A package.json whose + * only change is the version stamp is neither selected nor a canary trigger. + */ +export function prSelectionCost(diff: PrDiff, deps: SelectionDeps, table: CaseCostTable, options: { canaryUsd?: number } = {}): PrSelectionCost { + const full = fullGateUsd(deps, table); + const gate = new Set(deps.gateCases); + const stampOnly = (file: string) => file === 'package.json' && diff.packageVersionOnly; + const today = diff.files.length ? deps.today(diff.files, diff.packageVersionOnly) : { e2e: [], judges: [], shadowE2e: null, mode: 'none' }; + const shadowE2e = today.shadowE2e ?? today.e2e; + const preambleOnly = heuristicPreambleOnly(Object.fromEntries(Object.entries(diff.skillMdChangedLines).filter(([file]) => diff.files.includes(file)))); + const screenFiles = diff.files.filter(file => !preambleOnly.includes(file) && !stampOnly(file)); + const touch = screenFiles.length ? deps.touchfiles(screenFiles) : { e2e: [], global: false }; + const screenE2e = touch.e2e.filter(id => gate.has(id)); + const canaryReasons = [ + ...(preambleOnly.length ? [`${preambleOnly.length} preamble-only SKILL.md file(s) (heuristic)`] : []), + ...diff.files.filter(file => (SCREEN_FULL_GATE_FILES as readonly string[]).includes(file) && !stampOnly(file)).map(file => `full-gate file ${file}`), + ...diff.files.filter(file => file.startsWith('scripts/resolvers/')).map(file => `resolver ${file}`), + ...(today.e2e.length > screenE2e.length ? ['transitive-only files (today selects more cases than declared touchfiles)'] : []), + ]; + const canary = canaryReasons.length > 0; + const screenUsd = costOf(screenE2e, table) + (canary && !touch.global ? options.canaryUsd ?? 12 : 0) + costOf(deps.judgeCases, table); + return { + pr: diff.pr, sha: diff.sha, files: diff.files.length, mode: today.mode, + today: { cases: today.e2e.length, judges: today.judges.length, usd: costOf(today.e2e, table) + costOf(today.judges, table) }, + shadow: { cases: shadowE2e.length, judges: today.judges.length, usd: costOf(shadowE2e, table) + costOf(today.judges, table) }, + screenProvisional: { cases: screenE2e.length, judges: deps.judgeCases.length, usd: Math.min(screenUsd, full), canary, canaryReasons, preambleOnly }, + }; +} + +export interface SelectorSummary { meanUsd: number; medianUsd: number; meanCases: number; nearFull: number } + +/** One cohort: every merged PR counts, zero-cost PRs included. The median is the upper middle value. */ +export function summarizeSelector(rows: SelectorCost[], fullUsd: number): SelectorSummary { + const usd = rows.map(row => row.usd).sort((a, b) => a - b); + const n = Math.max(1, rows.length); + return { + meanUsd: usd.reduce((a, b) => a + b, 0) / n, + medianUsd: usd[usd.length >> 1] ?? 0, + meanCases: rows.reduce((sum, row) => sum + row.cases, 0) / n, + nearFull: usd.filter(value => value >= 0.9 * fullUsd).length, + }; +} + +// --- 4. Merge-train model --- + +/** CPython's random.Random (MT19937 with init_by_array seeding, random(), shuffle(), sample()), so a seed reproduces sim.py. */ +export class PyRandom { + private readonly mt = new Uint32Array(624); + private index = 624; + + constructor(seed: number) { + if (!Number.isSafeInteger(seed) || seed < 0) throw new Error('seed must be a non-negative safe integer'); + const key: number[] = []; + for (let rest = seed; key.length === 0 || rest > 0; rest = Math.floor(rest / 2 ** 32)) key.push(rest % 2 ** 32); + const mt = this.mt; + mt[0] = 19650218; + for (let i = 1; i < 624; i += 1) mt[i] = Math.imul(0x6c078965, mt[i - 1]! ^ (mt[i - 1]! >>> 30)) + i; + let i = 1, j = 0; + for (let k = Math.max(624, key.length); k > 0; k -= 1) { + mt[i] = (mt[i]! ^ Math.imul(mt[i - 1]! ^ (mt[i - 1]! >>> 30), 0x0019660d)) + key[j]! + j; + i += 1; j += 1; + if (i >= 624) { mt[0] = mt[623]!; i = 1; } + if (j >= key.length) j = 0; + } + for (let k = 623; k > 0; k -= 1) { + mt[i] = (mt[i]! ^ Math.imul(mt[i - 1]! ^ (mt[i - 1]! >>> 30), 0x5d588b65)) - i; + i += 1; + if (i >= 624) { mt[0] = mt[623]!; i = 1; } + } + mt[0] = 0x80000000; + } + + private next32(): number { + const mt = this.mt; + if (this.index >= 624) { + for (let k = 0; k < 624; k += 1) { + const y = (mt[k]! & 0x80000000) | (mt[(k + 1) % 624]! & 0x7fffffff); + mt[k] = mt[(k + 397) % 624]! ^ (y >>> 1) ^ (y & 1 ? 0x9908b0df : 0); + } + this.index = 0; + } + let y = mt[this.index++]!; + y ^= y >>> 11; + y ^= (y << 7) & 0x9d2c5680; + y ^= (y << 15) & 0xefc60000; + y ^= y >>> 18; + return y >>> 0; + } + + random(): number { + const a = this.next32() >>> 5, b = this.next32() >>> 6; + return (a * 67108864 + b) / 9007199254740992; + } + + /** CPython's _randbelow: rejection sampling on n.bit_length() random bits. */ + private below(n: number): number { + const bits = n.toString(2).length; + let r = this.next32() >>> (32 - bits); + while (r >= n) r = this.next32() >>> (32 - bits); + return r; + } + + shuffle(items: T[]): void { + for (let i = items.length - 1; i > 0; i -= 1) { + const j = this.below(i + 1); + [items[i], items[j]] = [items[j]!, items[i]!]; + } + } + + sample(population: readonly T[], k: number): T[] { + const n = population.length; + if (!Number.isInteger(k) || k < 0 || k > n) throw new Error('sample larger than population or negative'); + const setsize = 21 + (k > 5 ? 4 ** Math.ceil(Math.log(k * 3) / Math.log(4)) : 0); + const result: T[] = []; + if (n <= setsize) { + const pool = [...population]; + for (let i = 0; i < k; i += 1) { + const j = this.below(n - i); + result.push(pool[j]!); + pool[j] = pool[n - i - 1]!; + } + return result; + } + const picked = new Set(); + for (let i = 0; i < k; i += 1) { + let j = this.below(n); + while (picked.has(j)) j = this.below(n); + picked.add(j); + result.push(population[j]!); + } + return result; + } +} + +export type ScreenPolicy = + | { kind: 'all' } | { kind: 'none' } | { kind: 'random'; fraction: number } + | { kind: 'cheapest-half' } | { kind: 'expensive-half' }; + +export function policyLabel(policy: ScreenPolicy): string { + return policy.kind === 'random' ? `random ${Math.round(policy.fraction * 100)}%` : policy.kind; +} + +export interface TrainModelInput { + /** Every merged PR of the window; `screenUsd` is its replayed screen cost on the recorded full-gate scale. */ + prs: Array<{ paidRelevant: boolean; screenUsd: number }>; + policy: ScreenPolicy; + windowDays: number; + trainSize?: number; + outsideRed?: number; + redScreened?: number; + redUnscreened?: number; + fullGateUsd?: number; + bisectUsd?: number; + rescreenFactor?: number; + reps?: number; + seed?: number; + todayWeeklyUsd?: number; + afterGsta48WeeklyUsd?: number; +} + +export interface TrainModelResult { + policy: string; + trainSize: number; + outsideRed: number; + weeklyUsd: number; + perMergedPrUsd: number; + multipleVsToday: number; + multipleVsAfterGsta48: number; + screenWeeklyUsd: number; + initialTrainsWeekly: number; + gateAttemptsWeekly: number; + bisectsWeekly: number; + rebuildOnceWeekly: number; + stallsWeekly: number; + firstCandidateRedShare: number; + attemptRedShare: number; +} + +/** + * Every merged PR lands in a train. Each paid-relevant PR carries a defect at + * the screened or unscreened rate, and each gate attempt can also flake + * (outside red). A red attempt pays a bisect. A defect is attributed to the + * train's first defective PR, which is dropped, repaired and requeued at the + * back; a red with no defect gets one unchanged rebuild, and a second such + * red counts as a stall (and is rebuilt again). + */ +export function simulateTrains(input: TrainModelInput): TrainModelResult { + const { prs, policy, windowDays } = input; + const k = input.trainSize ?? 5, o = input.outsideRed ?? 0.15, rs = input.redScreened ?? 0.06, ru = input.redUnscreened ?? 0.25; + const full = input.fullGateUsd ?? 93.14, bisect = input.bisectUsd ?? 6, rescreen = input.rescreenFactor ?? 1.3; + const reps = input.reps ?? 400; + const rnd = new PyRandom(input.seed ?? 1); + const relevant = prs.map((pr, index) => ({ ...pr, index })).filter(pr => pr.paidRelevant); + const byCost = [...relevant].sort((a, b) => a.screenUsd - b.screenUsd); + const half = relevant.length >> 1; + const t = { screen: 0, gates: 0, init: 0, initRed: 0, attRed: 0, bisects: 0, rebuildOnce: 0, stalls: 0 }; + for (let rep = 0; rep < reps; rep += 1) { + const chosen = policy.kind === 'all' ? relevant : policy.kind === 'none' ? [] + : policy.kind === 'random' ? rnd.sample(relevant, Math.floor(relevant.length * policy.fraction)) + : policy.kind === 'cheapest-half' ? byCost.slice(0, half) : byCost.slice(half); + const screened = new Set(chosen.map(pr => pr.index)); + const queue = prs.map((pr, index) => { + const s = screened.has(index); + if (s) t.screen += pr.screenUsd * rescreen; + return { defect: pr.paidRelevant && rnd.random() < (s ? rs : ru) }; + }); + rnd.shuffle(queue); + while (queue.length) { + let train = queue.splice(0, k); + t.init += 1; + let first = true, rebuilt = false; + while (true) { + t.gates += 1; + const flake = rnd.random() < o; + const bad = train.filter(item => item.defect); + const red = flake || bad.length > 0; + if (first && red) t.initRed += 1; + first = false; + if (!red) break; + t.attRed += 1; + t.bisects += 1; + if (bad.length) { + const culprit = bad[0]!; + train = train.filter(item => item !== culprit); + culprit.defect = false; + queue.push(culprit); + rebuilt = false; + if (!train.length) break; + } else if (!rebuilt) { + rebuilt = true; + t.rebuildOnce += 1; + } else { + t.stalls += 1; + rebuilt = false; + } + } + } + } + const perWeek = 7 / windowDays / reps; + const weeklyUsd = (t.screen + t.gates * full + t.bisects * bisect) * perWeek; + return { + policy: policyLabel(policy), trainSize: k, outsideRed: o, weeklyUsd, + perMergedPrUsd: weeklyUsd / (prs.length * 7 / windowDays), + multipleVsToday: (input.todayWeeklyUsd ?? TODAY_WEEKLY_USD) / weeklyUsd, + multipleVsAfterGsta48: (input.afterGsta48WeeklyUsd ?? AFTER_GSTA48_WEEKLY_USD) / weeklyUsd, + screenWeeklyUsd: t.screen * perWeek, initialTrainsWeekly: t.init * perWeek, gateAttemptsWeekly: t.gates * perWeek, + bisectsWeekly: t.bisects * perWeek, rebuildOnceWeekly: t.rebuildOnce * perWeek, stallsWeekly: t.stalls * perWeek, + firstCandidateRedShare: t.init ? t.initRed / t.init : 0, attemptRedShare: t.gates ? t.attRed / t.gates : 0, + }; +} diff --git a/package.json b/package.json index 75a61bf43..e98f86e9c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.136", + "version": "1.91.137", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/gate-cost-replay.ts b/scripts/gate-cost-replay.ts new file mode 100644 index 000000000..856ed6ced --- /dev/null +++ b/scripts/gate-cost-replay.ts @@ -0,0 +1,358 @@ +#!/usr/bin/env bun +/** + * gate-cost-replay — what the paid PR gate costs and what each selection + * policy would have cost on recent merges (GSTA-49 plan lane T0). Free and + * read-only: it downloads recorded report artifacts with `gh` into a cache, + * replays origin/main's first-parent squash merges through the selectors in + * this checkout, and models merge trains. Logic lives in lib/gate-cost.ts. + * + * Usage: + * bun run scripts/gate-cost-replay.ts --runs [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202] + * [--ref origin/main] [--repo owner/name] [--cache .context/gate-cost-cache] [--window-days N] + * [--usd-per-second R] + * [--train-size 5] [--reps 400] [--seed 1] [--cases] [--json] + * + * --runs names complete full-gate runs (108 rule cases + 24 judges). Bare + * dates are UTC days; --until is inclusive. Cases without a recorded cost are + * estimated from their duration (default rate: recorded $ over recorded + * session-seconds; --usd-per-second pins it). Screening figures are labelled + * screen-provisional: the preamble detector is a heuristic until T2 lands. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + SCREEN_LABEL, TODAY_WEEKLY_USD, AFTER_GSTA48_WEEKLY_USD, caseCostTable, changedLinesByFile, fullGateUsd, isSkillMd, + parseEvalReport, parseTrialOutcomes, prSelectionCost, simulateTrains, summarizeGates, summarizeSelector, + type CaseCostEntry, type GateRecord, type GateSummary, type PrDiff, type PrSelectionCost, type ScreenPolicy, + type SelectionDeps, type SelectorSummary, type TrainModelResult, +} from '../lib/gate-cost'; +import { GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history'; +import { computePaidCaseSelection } from './lib/paid-select'; +import { packageChangeOnlyVersion } from './test-pr-profile'; +import { E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const DAY_MS = 86_400_000; +const ARTIFACT = /^(paid-slice-\d+|trial-outcomes[A-Za-z0-9_-]*?)-a(\d+)$/; +export const MODEL_POLICIES: ScreenPolicy[] = [ + { kind: 'all' }, { kind: 'random', fraction: 0.8 }, { kind: 'random', fraction: 0.5 }, + { kind: 'cheapest-half' }, { kind: 'expensive-half' }, { kind: 'none' }, +]; +export const MODEL_OUTSIDE_RED = [0.10, 0.15]; + +export interface ReplayArgs { + runs: string[]; + since?: string; + until?: string; + prs: number[]; + ref: string; + repo: string; + cache: string; + windowDays?: number; + usdPerSecond?: number; + trainSize: number; + reps: number; + seed: number; + cases: boolean; + json: boolean; +} + +export function parseArgs(argv: string[]): ReplayArgs { + const args: ReplayArgs = { runs: [], prs: [], ref: 'origin/main', repo: '', cache: path.join(ROOT, '.context', 'gate-cost-cache'), + trainSize: 5, reps: 400, seed: 1, cases: false, json: false }; + const list = (value: string) => value.split(/[\s,]+/).filter(Boolean); + const positive = (flag: string, value: string) => { + const n = Number(value); + if (!Number.isFinite(n) || n <= 0) throw new Error(`${flag} needs a positive number, got ${value}`); + return n; + }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]!; + if (flag === '--json') { args.json = true; continue; } + if (flag === '--cases') { args.cases = true; continue; } + const value = argv[i + 1]; + if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value`); + i += 1; + if (flag === '--runs') args.runs.push(...list(value)); + else if (flag === '--prs') args.prs.push(...list(value).map(pr => Number(pr.replace(/^#/, '')))); + else if (flag === '--since') args.since = value; + else if (flag === '--until') args.until = value; + else if (flag === '--ref') args.ref = value; + else if (flag === '--repo') args.repo = value; + else if (flag === '--cache') args.cache = path.resolve(value); + else if (flag === '--window-days') args.windowDays = positive(flag, value); + else if (flag === '--usd-per-second') args.usdPerSecond = positive(flag, value); + else if (flag === '--train-size') args.trainSize = Math.round(positive(flag, value)); + else if (flag === '--reps') args.reps = Math.round(positive(flag, value)); + else if (flag === '--seed') args.seed = Math.round(Number(value)); + else throw new Error(`unknown flag ${flag}`); + } + if (args.runs.length === 0) throw new Error('--runs is required: the full-gate runs whose recorded costs price every case'); + if (args.runs.some(run => !/^\d+$/.test(run))) throw new Error('--runs takes numeric GitHub Actions run ids'); + if (args.prs.some(pr => !Number.isInteger(pr) || pr <= 0)) throw new Error('--prs takes PR numbers'); + if (args.prs.length && (args.since || args.until)) throw new Error('use either --prs or --since/--until, not both'); + if (!!args.since !== !!args.until) throw new Error('--since and --until go together'); + return args; +} + +/** A bare date is a UTC day; `end` makes it inclusive (the next midnight). */ +export function parseWindowDate(value: string, end: boolean): Date { + const bare = /^\d{4}-\d{2}-\d{2}$/.test(value); + const date = new Date(bare ? `${value}T00:00:00Z` : value); + if (Number.isNaN(date.getTime())) throw new Error(`not a date: ${value}`); + return bare && end ? new Date(date.getTime() + DAY_MS) : date; +} + +function git(args: string[], timeout = 60_000): string { + const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 }); + if (result.status !== 0) throw new Error(`git ${args.slice(0, 3).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`); + return result.stdout; +} + +// --- Artifacts --- + +/** Download a run's paid-slice and trial-outcomes artifacts once; a run with a .complete marker never calls gh again. */ +export function fetchRun(runId: string, cacheDir: string, repo: string): string { + const dir = path.join(cacheDir, runId); + if (fs.existsSync(path.join(dir, '.complete'))) return dir; + downloadRunArtifacts({ repo, run: { id: Number(runId) }, cacheDir, match: name => ARTIFACT.test(name), maxBytes: 512 * 1024 * 1024, fetcher: GH_HISTORY }); + fs.writeFileSync(path.join(dir, '.complete'), ''); + return dir; +} + +/** One run directory -> GateRecord, reading only each artifact's highest attempt. */ +export function readGateRecord(runDir: string, runId = path.basename(runDir)): GateRecord { + const latest = new Map(); + for (const name of fs.existsSync(runDir) ? fs.readdirSync(runDir) : []) { + const match = name.match(ARTIFACT); + if (!match || (latest.get(match[1]!)?.attempt ?? 0) >= Number(match[2])) continue; + latest.set(match[1]!, { name, attempt: Number(match[2]) }); + } + const gate: GateRecord = { runId, tests: [], trials: [] }; + for (const { name } of [...latest.values()].sort((a, b) => a.name.localeCompare(b.name))) { + const artifact = path.join(runDir, name); + if (name.startsWith('trial-outcomes')) { + const file = path.join(artifact, 'trial-outcomes.jsonl'); + if (fs.existsSync(file)) gate.trials.push(...parseTrialOutcomes(fs.readFileSync(file, 'utf8'))); + continue; + } + const shards = path.join(artifact, 'shards'); + for (const shard of fs.existsSync(shards) ? fs.readdirSync(shards).sort() : []) { + const shardDir = path.join(shards, shard); + if (!fs.statSync(shardDir).isDirectory()) continue; + for (const file of fs.readdirSync(shardDir).sort()) { + if (!file.endsWith('.json') || file.startsWith('_partial') || file.startsWith('slice-')) continue; + let report: unknown; + try { report = JSON.parse(fs.readFileSync(path.join(shardDir, file), 'utf8')); } catch { continue; } + gate.tests.push(...parseEvalReport(shard, report)); + } + } + } + return gate; +} + +// --- Merged PRs --- + +export interface MergedPr { pr: number; sha: string; committedAt: string } + +/** First-parent squash merges whose subject ends in (#NNN). */ +export function parseMergeLog(log: string): MergedPr[] { + return log.split('\n').filter(Boolean).flatMap(line => { + const [sha, committedAt, ...subject] = line.split('\t'); + const match = subject.join('\t').match(/\(#(\d+)\)\s*$/); + return match && sha && committedAt ? [{ pr: Number(match[1]), sha, committedAt }] : []; + }); +} + +function mergedPrs(args: ReplayArgs): { prs: MergedPr[]; windowDays: number | null; since?: Date; until?: Date } { + const format = '--format=%H%x09%cI%x09%s'; + const shallowHint = (since: string) => git(['rev-parse', '--is-shallow-repository']).trim() === 'true' + ? `; this checkout is shallow: run "git fetch --shallow-since=${since} origin main" and rerun` : ''; + if (args.since && args.until) { + const since = parseWindowDate(args.since, false), until = parseWindowDate(args.until, true); + const oldest = git(['log', '--first-parent', '--format=%cI', args.ref]).trim().split('\n').pop() ?? ''; + const hint = shallowHint(since.toISOString().slice(0, 10)); + if (hint && new Date(oldest) > since) throw new Error(`history before ${oldest} is missing${hint}`); + const prs = parseMergeLog(git(['log', '--first-parent', `--since=${since.toISOString()}`, `--until=${until.toISOString()}`, format, args.ref])).reverse(); + return { prs, windowDays: args.windowDays ?? (until.getTime() - since.getTime()) / DAY_MS, since, until }; + } + if (args.prs.length === 0) return { prs: [], windowDays: null }; + const all = new Map(parseMergeLog(git(['log', '--first-parent', format, args.ref])).map(pr => [pr.pr, pr])); + const missing = args.prs.filter(pr => !all.has(pr)); + if (missing.length) throw new Error(`no squash merge of #${missing.join(', #')} on ${args.ref}'s first-parent history${shallowHint('2026-01-01')}`); + return { prs: args.prs.map(pr => all.get(pr)!), windowDays: args.windowDays ?? null }; +} + +function prDiff(merge: MergedPr): PrDiff { + const files = git(['diff', '--name-only', '--no-renames', `${merge.sha}^`, merge.sha]).split('\n').filter(Boolean); + let packageVersionOnly = false; + if (files.includes('package.json')) { + const show = (rev: string) => spawnSync('git', ['show', `${rev}:package.json`], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + const before = show(`${merge.sha}^`), after = show(merge.sha); + packageVersionOnly = before.status === 0 && after.status === 0 && packageChangeOnlyVersion(before.stdout, after.stdout); + } + const skillMds = files.filter(isSkillMd); + const skillMdChangedLines = skillMds.length >= 10 + ? changedLinesByFile(git(['diff', '-U0', '--no-renames', `${merge.sha}^`, merge.sha, '--', ...skillMds], 120_000)) : {}; + return { pr: merge.pr, sha: merge.sha.slice(0, 8), files, packageVersionOnly, skillMdChangedLines }; +} + +/** The selectors in this checkout: today's PR profile (with R4's shadow) and the plain touchfile selection. */ +export function checkoutSelectionDeps(rootDir = ROOT): SelectionDeps { + const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate'); + const judgeCases = Object.keys(LLM_JUDGE_TOUCHFILES); + return { + gateCases, judgeCases, + today(files, packageVersionOnly) { + try { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, rootDir, changedFiles: files, packageVersionOnly }); + return { e2e: result.selection.e2e ?? gateCases, judges: result.selection.judges ?? judgeCases, + shadowE2e: result.coverage?.shadow?.e2e ?? null, mode: result.coverage?.mode ?? 'pr' }; + } catch { + return { e2e: gateCases, judges: judgeCases, shadowE2e: null, mode: 'needs-full' }; + } + }, + touchfiles(files) { + const result = computePaidCaseSelection({ profile: 'full', env: {}, rootDir, changedFiles: files }); + return { e2e: result.selection.e2e ?? gateCases, global: files.some(file => GLOBAL_TOUCHFILES.some(pattern => matchGlob(file, pattern))) }; + }, + }; +} + +// --- Report --- + +export interface ReplayReport { + repo: string; + ref: string; + head: string; + runs: Array<{ runId: string; tests: number; trials: number; recordedUsd: number }>; + gate: GateSummary; + fullScaleUsd: number; + usdPerSecond: number; + cases: CaseCostEntry[]; + window: { since: string; until: string } | null; + windowDays: number | null; + prs: PrSelectionCost[]; + selection: { today: SelectorSummary; shadow: SelectorSummary; screenProvisional: SelectorSummary } | null; + model: { screenScale: number; fullGateUsd: number; trainSize: number; reps: number; seed: number; rows: TrainModelResult[] } | null; +} + +export function buildReport(args: ReplayArgs, gates: GateRecord[], merges: ReturnType, deps: SelectionDeps, + diffs: (merge: MergedPr) => PrDiff = prDiff): Omit { + const table = caseCostTable(gates, { usdPerSecond: args.usdPerSecond }); + const gate = summarizeGates(gates); + const fullScaleUsd = fullGateUsd(deps, table); + const prs = merges.prs.map(merge => prSelectionCost(diffs(merge), deps, table)); + const selection = prs.length ? { + today: summarizeSelector(prs.map(row => row.today), fullScaleUsd), + shadow: summarizeSelector(prs.map(row => row.shadow), fullScaleUsd), + screenProvisional: summarizeSelector(prs.map(row => row.screenProvisional), fullScaleUsd), + } : null; + const screenScale = fullScaleUsd ? gate.recordedUsd.mean / fullScaleUsd : 0; + const model = prs.length && merges.windowDays ? { + screenScale, fullGateUsd: gate.recordedUsd.mean, trainSize: args.trainSize, reps: args.reps, seed: args.seed, + rows: MODEL_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateTrains({ + prs: prs.map(row => ({ paidRelevant: row.today.usd > 0, screenUsd: row.screenProvisional.usd * screenScale })), + policy, outsideRed, windowDays: merges.windowDays!, trainSize: args.trainSize, fullGateUsd: gate.recordedUsd.mean, reps: args.reps, seed: args.seed, + }))), + } : null; + return { + runs: gates.map(g => ({ runId: g.runId, tests: g.tests.length, trials: g.trials.length, recordedUsd: g.tests.reduce((sum, t) => sum + t.costUsd, 0) })), + gate, fullScaleUsd, usdPerSecond: table.usdPerSecond, cases: Object.values(table.cases).sort((a, b) => b.costUsd - a.costUsd), + window: merges.since && merges.until ? { since: merges.since.toISOString(), until: merges.until.toISOString() } : null, + windowDays: merges.windowDays, prs, selection, model, + }; +} + +const usd = (value: number) => value.toFixed(2); +const pct = (value: number, total: number) => (total ? `${(100 * value / total).toFixed(1)}%` : '-'); +const mtok = (tokens: number) => (tokens >= 1e6 ? `${(tokens / 1e6).toFixed(2)}M` : `${Math.round(tokens / 1e3)}k`); +const pad = (cells: Array, widths: number[]) => cells.map((cell, i) => String(cell).padEnd(widths[i] ?? 0)).join(' ').trimEnd(); + +export function formatReport(report: ReplayReport, options: { cases?: boolean } = {}): string { + const out: string[] = []; + const { gate } = report; + const gateUsd = gate.recordedUsd.mean; + out.push(`gate-cost replay: ${report.repo} at ${report.head.slice(0, 8)} (selectors from this checkout)`, ''); + out.push(`Gates: ${report.runs.length} run(s), recorded $ per gate mean ${usd(gateUsd)} (min ${usd(gate.recordedUsd.min)}, max ${usd(gate.recordedUsd.max)})`); + for (const run of report.runs) out.push(` run ${run.runId}: ${run.trials} trials, ${run.tests} cost entries, $${usd(run.recordedUsd)}`); + out.push('', 'Recorded $ per gate by model:'); + for (const [model, value] of Object.entries(gate.byModel).sort((a, b) => b[1] - a[1])) out.push(` ${pad([model, usd(value), pct(value, gateUsd)], [34, 8])}`); + out.push('', 'Token classes per gate (lib/pricing.ts prices; cache writes at 1.25x input; share of the recorded gate):'); + const classes = Object.values(gate.tokenClasses); + const sum = (pick: (c: (typeof classes)[number]) => number) => classes.reduce((total, c) => total + pick(c), 0); + const thinking = sum(c => c.output.thinkingTokens); + for (const [label, key] of [['cache write', 'cacheWrite'], [`output (incl. ${mtok(thinking)} thinking)`, 'output'], ['cache read', 'cacheRead'], ['uncached input', 'uncachedInput']] as const) { + const value = sum(c => c[key].usd); + out.push(` ${pad([label, mtok(sum(c => c[key].tokens)), usd(value), pct(value, gateUsd)], [34, 8, 8])}`); + } + for (const [model, c] of Object.entries(gate.tokenClasses)) { + out.push(` ${model}: ${c.priced ? `priced $${usd(c.pricedUsd)} vs recorded $${usd(c.recordedUsd)}` : `no lib/pricing.ts row, recorded $${usd(c.recordedUsd)} unpriced`}`); + } + const estimated = report.cases.filter(c => c.estimated); + const estimatedUsd = estimated.reduce((total, c) => total + c.costUsd, 0); + out.push('', `Cases: ${report.cases.length}; ${estimated.length} without recorded cost, estimated at $${report.usdPerSecond.toFixed(4)} per session-second: ~${usd(estimatedUsd)} per gate`); + for (const c of estimated) out.push(` ${pad([c.caseId, `~${usd(c.costUsd)} (est.)`, `${Math.round(c.meanDurationMs / 1000)} s`, c.runner ?? '-'], [40, 14, 7])}`); + out.push(`Full-gate scale: $${usd(report.fullScaleUsd)} per gate (recorded case means plus estimates)`); + if (options.cases) { + out.push('', 'Per-case mean cost:'); + for (const c of report.cases) out.push(` ${pad([c.caseId, (c.estimated ? '~' : '') + usd(c.costUsd), c.kind, c.runner ?? '-', c.estimated ? 'estimated' : `${c.recordedGates} gate(s)`], [50, 8, 6, 10])}`); + } + if (!report.prs.length) return out.join('\n'); + const label = SCREEN_LABEL; + out.push('', `Per merged PR ($ on the $${usd(report.fullScaleUsd)} full-gate scale; cases are gate e2e cases, judges extra):`); + out.push(` ${pad(['PR', 'sha', 'files', 'today $ (cases)', 'R4 shadow $ (cases)', `${label} $ (cases)`], [6, 8, 5, 16, 20])}`); + for (const row of report.prs) { + const cell = (s: { usd: number; cases: number }) => `${usd(s.usd)} (${s.cases})`; + out.push(` ${pad([`#${row.pr}`, row.sha, row.files, cell(row.today), cell(row.shadow), cell(row.screenProvisional) + (row.screenProvisional.canary ? ' +canary' : '')], [6, 8, 5, 16, 20])}`); + } + const s = report.selection!; + out.push('', `Selection over ${report.prs.length} merged PRs (one cohort: every merged PR counts, zeros included)${report.window ? `, ${report.window.since} to ${report.window.until}` : ''}:`); + out.push(` ${pad(['selector', '$ / merged PR', 'median $', 'mean cases', 'PRs >= 90% of gate'], [20, 13, 9, 10])}`); + for (const [name, row] of [['today', s.today], ['R4 shadow', s.shadow], [label, s.screenProvisional]] as const) { + out.push(` ${pad([name, usd(row.meanUsd), usd(row.medianUsd), row.meanCases.toFixed(1), row.nearFull], [20, 13, 9, 10])}`); + } + out.push(` ${label}: its preamble detector is a line-commonality heuristic, not T2's hunk classifier; heuristic until T2 lands.`); + if (!report.model) { + out.push('', 'Train model: skipped (needs a window: --since/--until, or --window-days with --prs).'); + return out.join('\n'); + } + const m = report.model; + const perWeek = report.prs.length * 7 / report.windowDays!; + out.push('', `Train model (seeded Monte Carlo, ${m.reps} reps, seed ${m.seed}; trains of ${m.trainSize}; full gate $${usd(m.fullGateUsd)}, bisect $6, repair re-screens x1.3,`, + ` red rates screened 6% / unscreened 25%; ${label} screen costs scaled by ${m.screenScale.toFixed(4)} to the recorded gate;`, + ` window ${report.windowDays!.toFixed(1)} days, ${perWeek.toFixed(1)} merged PRs a week; multiples against $${TODAY_WEEKLY_USD} and $${AFTER_GSTA48_WEEKLY_USD} a week):`); + const widths = [15, 5, 7, 6, 7, 7, 22, 9, 9, 9, 8, 9, 10]; + out.push(` ${pad(['policy', 'o', '$/week', '$/PR', 'x9950', 'x8400', `${label} $/w`, 'trains/w', 'gates/w', 'bisect/w', 'rebld/w', 'stalls/w', 'first-red'], widths)} attempt-red`); + for (const r of m.rows) { + out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), Math.round(r.weeklyUsd), r.perMergedPrUsd.toFixed(1), r.multipleVsToday.toFixed(2), r.multipleVsAfterGsta48.toFixed(2), + Math.round(r.screenWeeklyUsd), r.initialTrainsWeekly.toFixed(2), r.gateAttemptsWeekly.toFixed(2), r.bisectsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2), + r.stallsWeekly.toFixed(2), pct(r.firstCandidateRedShare, 1)], widths)} ${pct(r.attemptRedShare, 1)}`); + } + return out.join('\n'); +} + +export function main(argv: string[]): number { + let args: ReplayArgs; + try { args = parseArgs(argv); } catch (error) { + console.error(`gate-cost-replay: ${(error as Error).message}`); + return 2; + } + try { + const repo = args.repo || repoSlug(ROOT); + fs.mkdirSync(args.cache, { recursive: true }); + const gates = args.runs.map(run => readGateRecord(fetchRun(run, args.cache, repo), run)); + const empty = gates.filter(g => g.trials.length === 0); + if (empty.length) throw new Error(`run(s) ${empty.map(g => g.runId).join(', ')} have no trial outcomes (expired artifacts, or not a paid gate run)`); + const merges = mergedPrs(args); + const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...buildReport(args, gates, merges, checkoutSelectionDeps()) }; + console.log(args.json ? JSON.stringify(report, null, 2) : formatReport(report, { cases: args.cases })); + return 0; + } catch (error) { + console.error(`gate-cost-replay: ${(error as Error).message}`); + return 1; + } +} + +if (import.meta.main) process.exit(main(process.argv.slice(2))); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index 6a6555c98..04ef8b134 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -121,7 +121,9 @@ { "file": "lib/script-invocation.ts", "movedFrom": "(new: the one Windows bash resolver, shebang-bin invocation and wtreeFingerprint, from spawn-bin.ts and gbrain-exec.ts, #3197)" }, { "file": "lib/autoplan-budget.ts", "movedFrom": "(new: round cap, default host estimate, budget_exhausted stop and the budget verb, field list 2 W5)" }, { "file": "lib/autoplan-confirm.ts", "movedFrom": "(new: confirm-round plan delta, revision-qualified correlation and settled findings, field list 2 W5)" }, - { "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" } + { "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" }, + { "file": "lib/gate-cost.ts", "movedFrom": "(new: paid gate cost by model and token class, per-case costs, selection replays and the merge-train model, GSTA-49 T0)" }, + { "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" } ], "residualFiles": [ { "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" }, diff --git a/test/gate-cost.test.ts b/test/gate-cost.test.ts new file mode 100644 index 000000000..3d5092b4a --- /dev/null +++ b/test/gate-cost.test.ts @@ -0,0 +1,221 @@ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + PyRandom, SCREEN_LABEL, caseCostTable, changedLinesByFile, gateCostBreakdown, heuristicPreambleOnly, parseEvalReport, + prSelectionCost, simulateTrains, summarizeGates, summarizeSelector, type GateRecord, type PrDiff, type SelectionDeps, +} from '../lib/gate-cost'; +import { buildReport, checkoutSelectionDeps, formatReport, parseArgs, parseMergeLog, parseWindowDate, readGateRecord, type ReplayReport } from '../scripts/gate-cost-replay'; + +/** + * A one-run artifact tree, written to a temporary directory: tracked files + * under test/fixtures would need a FREE_FIXTURES entry, and that registry is + * a paid-runner input that selects every PR-profile case. + */ +const RUN_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-cost-run-')); +afterAll(() => fs.rmSync(RUN_DIR, { recursive: true, force: true })); +const usage = (inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number, thinkingTokens: number, costUSD: number) => + ({ inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens, thinkingTokens, costUSD }); +const FIXTURE_FILES: Record = { + 'paid-slice-1-a1/shards/skill-e2e-demo--case-a/1.0.0-HEAD-e2e-demo.json': { tests: [{ + name: '/case a', tier: 'e2e', duration_ms: 40_500, cost_usd: 0.253, model: 'claude-fable-5-1', transcript: [ + { type: 'assistant' }, + { type: 'result', modelUsage: { 'claude-fable-5-1': usage(100, 2000, 100_000, 10_000, 500, 0.251), 'claude-haiku-4-5-20251001': usage(1000, 100, 0, 0, 0, 0.0015) } }, + { type: 'result', modelUsage: { 'claude-fable-5-1': usage(50, 0, 0, 0, 0, 0.0005) } }, + ] }] }, + 'paid-slice-1-a1/shards/skill-e2e-demo--case-a/_partial-e2e.json': { tests: [{ name: '/case a', duration_ms: 40_500, cost_usd: 99 }] }, + 'paid-slice-2-a1/shards/skill-llm-eval/1.0.0-HEAD-llm-judge.json': { tests: [{ name: 'demo/SKILL.md judge', tier: 'llm-judge', duration_ms: 20_000, cost_usd: 0.06 }] }, +}; +for (const [file, body] of Object.entries(FIXTURE_FILES)) { + fs.mkdirSync(path.dirname(path.join(RUN_DIR, file)), { recursive: true }); + fs.writeFileSync(path.join(RUN_DIR, file), JSON.stringify(body)); +} +const trial = (caseId: string, file: string, kind: string, durationMs: number, runner?: string) => + JSON.stringify({ schema: 'gstack-trial-outcome/v1', case: caseId, file, kind, duration_ms: durationMs, cost_usd: 0, ...(runner ? { sessions: [{ runner }] } : {}) }); +fs.mkdirSync(path.join(RUN_DIR, 'trial-outcomes-pr-a1')); +fs.writeFileSync(path.join(RUN_DIR, 'trial-outcomes-pr-a1', 'trial-outcomes.jsonl'), [ + trial('case-a', 'test/skill-e2e-demo.test.ts', 'rule', 40_000, 'claude-p'), + trial('case-pty', 'test/skill-e2e-pty-demo.test.ts', 'rule', 100_000, 'pty'), + 'not json', + trial('demo/SKILL.md judge', 'test/skill-llm-eval.test.ts', 'judge', 20_000), +].join('\n') + '\n'); +const run1 = () => readGateRecord(RUN_DIR, '1'); + +/** GSTA-49's l3.json: 92 merged PRs of 2026-10-03..18, [today's selection cost > 0 ? 1 : 0, screen cost on the 106.0259 full-gate scale]. */ +const L3_ROWS: Array<[number, number]> = [ + [0,14.3731], [1,14.0509], [1,14.0509], [1,106.0259], [1,106.0259], [1,106.0259], [1,102.6497], [1,14.0509], [0,14.0509], [1,15.5642], + [0,15.5642], [0,0], [1,14.2545], [0,14.0509], [0,14.0509], [1,14.0509], [0,14.0509], [1,53.2204], [1,80.5128], [1,48.761], + [1,47.2944], [1,46.47], [0,15.5642], [1,86.2435], [1,14.0509], [1,50.8781], [1,27.4209], [1,57.6069], [1,106.0259], [1,14.0509], + [1,106.0259], [1,14.0509], [1,14.0509], [1,14.3731], [1,76.3276], [1,106.0259], [1,14.0509], [1,16.2643], [0,14.0509], [1,92.6175], + [1,78.0744], [1,14.6998], [1,31.99], [1,14.0509], [1,14.0509], [1,19.9311], [1,59.4491], [1,14.0509], [1,106.0259], [1,22.1538], + [1,14.0509], [1,78.4857], [1,62.5585], [1,106.0259], [1,14.0509], [1,50.6055], [1,79.9843], [1,79.5795], [1,67.6899], [0,0], + [1,106.0259], [1,14.0509], [1,14.0509], [1,14.6998], [1,95.9865], [1,14.6998], [0,15.5642], [1,20.58], [0,15.2423], [1,106.0259], + [1,33.2109], [1,106.0259], [1,106.0259], [0,15.5642], [0,14.0509], [0,15.5642], [1,106.0259], [1,106.0259], [1,106.0259], [0,15.5642], + [1,106.0259], [0,15.5642], [0,1.5133], [0,15.5642], [1,19.7931], [1,106.0259], [0,15.5642], [1,106.0259], [1,106.0259], [1,21.6093], + [1,106.0259], [1,106.0259], +]; + +describe('paid report artifacts', () => { + test('reads eval reports and trial outcomes, skipping partial reports', () => { + const gate = run1(); + expect(gate.tests.map(t => [t.name, t.file, t.costUsd])).toEqual([ + ['/case a', 'test/skill-e2e-demo.test.ts', 0.253], + ['demo/SKILL.md judge', 'test/skill-llm-eval.test.ts', 0.06], + ]); + expect(gate.tests[0]!.modelUsage['claude-fable-5-1']).toMatchObject({ inputTokens: 150, outputTokens: 2000, thinkingTokens: 500, costUSD: 0.2515 }); + expect(gate.trials.map(t => [t.caseId, t.kind, t.runner])).toEqual([['case-a', 'rule', 'claude-p'], ['case-pty', 'rule', 'pty'], ['demo/SKILL.md judge', 'judge', null]]); + expect(parseEvalReport('x', { notTests: [] })).toEqual([]); + }); + + test('aggregates cost by model and token class through lib/pricing.ts', () => { + const b = gateCostBreakdown(run1()); + expect(b.recordedUsd).toBeCloseTo(0.313, 9); + expect(b.byModel['claude-fable-5-1']).toBeCloseTo(0.2515, 9); + expect(b.byModel['claude-haiku-4-5-20251001']).toBeCloseTo(0.0015, 9); + expect(b.byModel['llm-judge']).toBeCloseTo(0.06, 9); + const fable = b.tokenClasses['claude-fable-5-1']!; + expect(fable.cacheWrite.usd).toBeCloseTo(0.125, 9); + expect(fable.output.usd).toBeCloseTo(0.1, 9); + expect(fable.output.thinkingTokens).toBe(500); + expect(fable.cacheRead.usd).toBeCloseTo(0.025, 9); + expect(fable.uncachedInput.usd).toBeCloseTo(0.0015, 9); + expect(fable.pricedUsd).toBeCloseTo(fable.recordedUsd, 9); + expect(b.tokenClasses['claude-haiku-4-5-20251001']!.pricedUsd).toBeCloseTo(0.0015, 9); + const twice = summarizeGates([run1(), { tests: run1().tests.map(t => ({ ...t, costUsd: t.costUsd * 3 })) }]); + expect(twice.recordedUsd.min).toBeCloseTo(0.313, 9); + expect(twice.recordedUsd.mean).toBeCloseTo(0.626, 9); + }); + + test('per-case mean across gates; a case with no recorded cost is estimated from duration and marked', () => { + const second: GateRecord = { ...run1(), tests: run1().tests.map(t => (t.name === '/case a' ? { ...t, costUsd: 0.453 } : t)) }; + const table = caseCostTable([run1(), second]); + expect(table.cases['case-a']).toMatchObject({ estimated: false, recordedGates: 2 }); + expect(table.cases['case-a']!.costUsd).toBeCloseTo(0.353, 9); + expect(table.usdPerSecond).toBeCloseTo((0.253 + 0.453 + 0.12) / 120, 9); + expect(table.cases['case-pty']).toMatchObject({ estimated: true, recordedGates: 0, runner: 'pty' }); + expect(table.cases['case-pty']!.costUsd).toBeCloseTo(100 * table.usdPerSecond, 9); + expect(caseCostTable([run1()], { usdPerSecond: 0.01 }).cases['case-pty']!.costUsd).toBeCloseTo(1, 9); + }); +}); + +const COSTS = { cases: Object.fromEntries([['a', 10], ['b', 5], ['c', 1], ['j1', 0.5], ['j2', 0.5]].map(([id, usd]) => [id, { caseId: id as string, costUsd: usd as number }])), usdPerSecond: 0 } as never; +function fakeDeps(touched: string[][] = []): SelectionDeps { + return { + gateCases: ['a', 'b', 'c'], judgeCases: ['j1', 'j2'], + today: files => ({ e2e: files.some(f => f.endsWith('.tmpl')) ? ['a', 'b'] : [], judges: [], shadowE2e: null, mode: 'pr' }), + touchfiles: files => { touched.push(files); return { e2e: files.some(f => f.endsWith('.tmpl')) ? ['b'] : [], global: false }; }, + }; +} +const diff = (files: string[], extra: Partial = {}): PrDiff => ({ pr: 1, sha: 'abc', files, packageVersionOnly: false, skillMdChangedLines: {}, ...extra }); + +describe('selection cost of a merged PR', () => { + test('a package.json version stamp is neither selected nor a canary trigger; every judge is always in the screen', () => { + const touched: string[][] = []; + const stamp = prSelectionCost(diff(['package.json', 'VERSION', 'CHANGELOG.md'], { packageVersionOnly: true }), fakeDeps(touched), COSTS); + expect(stamp.screenProvisional).toMatchObject({ canary: false, canaryReasons: [], cases: 0, judges: 2, usd: 1 }); + expect(touched).toEqual([['VERSION', 'CHANGELOG.md']]); + const real = prSelectionCost(diff(['package.json']), fakeDeps(), COSTS); + expect(real.screenProvisional.canaryReasons).toEqual(['full-gate file package.json']); + expect(real.screenProvisional.usd).toBe(13); + expect(prSelectionCost(diff([]), fakeDeps(), COSTS).screenProvisional.usd).toBe(1); + }); + + test('the version stamp also adds no canary through the checkout selectors', () => { + const row = prSelectionCost(diff(['package.json', 'VERSION', 'CHANGELOG.md'], { packageVersionOnly: true }), checkoutSelectionDeps(), COSTS); + expect(row.today.cases).toBe(0); + expect(row.screenProvisional.canary).toBe(false); + }); + + test('transitive-only files and heuristic preamble-only SKILL.md changes add the canary set, capped at the full gate', () => { + const row = prSelectionCost(diff(['review/SKILL.md.tmpl']), fakeDeps(), COSTS); + expect(row.today).toEqual({ cases: 2, judges: 0, usd: 15 }); + expect(row.shadow.usd).toBe(15); + expect(row.screenProvisional).toMatchObject({ cases: 1, usd: 17, canary: true }); + const skills = Array.from({ length: 10 }, (_, i) => `s${i}/SKILL.md`); + const lines = Object.fromEntries(skills.map((file, i) => [file, ['+shared preamble line', ...(i === 0 ? ['+only here'] : [])]])); + expect(heuristicPreambleOnly(lines)).toEqual(skills.slice(1)); + expect(heuristicPreambleOnly(Object.fromEntries(Object.entries(lines).slice(0, 9)))).toEqual([]); + const preamble = prSelectionCost(diff(skills, { skillMdChangedLines: lines }), fakeDeps(), COSTS); + expect(preamble.screenProvisional.preambleOnly).toHaveLength(9); + expect(preamble.screenProvisional.canaryReasons[0]).toContain('preamble-only'); + expect(prSelectionCost(diff(['setup', 'scripts/resolvers/x.ts']), { ...fakeDeps(), touchfiles: () => ({ e2e: ['a', 'b', 'c'], global: false }) }, COSTS).screenProvisional.usd).toBe(17); + }); + + test('git -U0 diffs parse into distinct changed lines per file', () => { + const text = ['diff --git a/x/SKILL.md b/x/SKILL.md', '--- a/x/SKILL.md', '+++ b/x/SKILL.md', '@@ -1 +1 @@', '-old line', '+new line ', '+new line', 'diff --git a/y/SKILL.md b/y/SKILL.md', '+other'].join('\n'); + expect(changedLinesByFile(text)).toEqual({ 'x/SKILL.md': ['-old line', '+new line'], 'y/SKILL.md': ['+other'] }); + }); + + test('selector summaries average over every merged PR, zeros included', () => { + expect(summarizeSelector([{ cases: 0, judges: 0, usd: 0 }, { cases: 2, judges: 0, usd: 10 }, { cases: 4, judges: 0, usd: 20 }], 20)) + .toEqual({ meanUsd: 10, medianUsd: 10, meanCases: 2, nearFull: 1 }); + }); +}); + +describe('merge-train model', () => { + test('PyRandom reproduces CPython random.Random', () => { + const r = new PyRandom(1); + expect([r.random(), r.random()]).toEqual([0.13436424411240122, 0.8474337369372327]); + const items = Array.from({ length: 10 }, (_, i) => i); + new PyRandom(1).shuffle(items); + expect(items).toEqual([6, 8, 9, 7, 5, 3, 0, 4, 1, 2]); + const s = new PyRandom(1); + expect(s.sample(Array.from({ length: 100 }, (_, i) => i), 3)).toEqual([17, 72, 97]); + expect(s.sample(Array.from({ length: 20 }, (_, i) => i), 10)).toEqual([2, 8, 3, 15, 14, 7, 10, 6, 17, 1]); + expect(new PyRandom(2 ** 40 + 7).random()).toBe(0.6137037779936511); + }); + + const prs = L3_ROWS.map(([relevant, cost]) => ({ paidRelevant: relevant === 1, screenUsd: cost * 93.14 / 106.0259 })); + const model = (policy: Parameters[0]['policy']) => simulateTrains({ prs, policy, windowDays: 15.1, trainSize: 5, outsideRed: 0.15 }); + + test('reproduces the measurement sim (trains of 5, o=0.15, 6%/25% red, 400 reps, seed 1)', () => { + expect(prs).toHaveLength(92); + const all = model({ kind: 'all' }), half = model({ kind: 'random', fraction: 0.5 }), none = model({ kind: 'none' }); + expect([all, half, none].map(r => Math.round(r.weeklyUsd))).toEqual([3403, 2672, 1986]); + expect(Math.round(all.screenWeeklyUsd)).toBe(2196); + expect(none.screenWeeklyUsd).toBe(0); + expect(all.gateAttemptsWeekly).toBeCloseTo(12.73, 2); + expect(none.bisectsWeekly).toBeCloseTo(10.27, 2); + expect(half.rebuildOnceWeekly).toBeCloseTo(1.44, 2); + expect(half.stallsWeekly).toBeCloseTo(0.22, 2); + expect(all.firstCandidateRedShare).toBeCloseTo(0.32, 2); + expect(none.attemptRedShare).toBeCloseTo(0.50, 2); + expect(all.multipleVsToday).toBeCloseTo(9950 / all.weeklyUsd, 9); + expect(all.multipleVsAfterGsta48).toBeCloseTo(8400 / all.weeklyUsd, 9); + expect(all.perMergedPrUsd).toBeCloseTo(all.weeklyUsd / (92 * 7 / 15.1), 9); + }); + + test('cheapest-half screens the cheaper half and is deterministic for a seed', () => { + const cheap = model({ kind: 'cheapest-half' }), expensive = model({ kind: 'expensive-half' }); + expect(Math.round(cheap.weeklyUsd)).toBe(1996); + expect(cheap.screenWeeklyUsd).toBeLessThan(expensive.screenWeeklyUsd); + expect(model({ kind: 'random', fraction: 0.8 })).toEqual(model({ kind: 'random', fraction: 0.8 })); + }); +}); + +describe('gate-cost-replay CLI', () => { + test('parses window dates as UTC days with an inclusive end, and merge subjects', () => { + expect(parseWindowDate('2026-10-03', false).toISOString()).toBe('2026-10-03T00:00:00.000Z'); + expect(parseWindowDate('2026-10-17', true).toISOString()).toBe('2026-10-18T00:00:00.000Z'); + expect(parseMergeLog('a1\t2026-10-17T00:00:00Z\tv1.0 fix: x (#3261)\nb2\t2026-10-17T00:00:00Z\tno pr here\n')).toEqual([{ pr: 3261, sha: 'a1', committedAt: '2026-10-17T00:00:00Z' }]); + expect(() => parseArgs(['--since', '2026-10-03', '--until', '2026-10-17'])).toThrow('--runs'); + expect(() => parseArgs(['--runs', '1', '--prs', '2', '--since', '2026-10-03', '--until', '2026-10-04'])).toThrow('either'); + }); + + test('every printed screening figure is labelled screen-provisional', () => { + const args = parseArgs(['--runs', '1', '--reps', '5']); + const merges = { prs: [{ pr: 7, sha: 'abcdef12', committedAt: '' }, { pr: 8, sha: 'bcdef123', committedAt: '' }], windowDays: 7 }; + const deps: SelectionDeps = { ...fakeDeps(), gateCases: ['case-a', 'case-pty'], judgeCases: ['demo/SKILL.md judge'] }; + const files: Record = { 7: ['x/SKILL.md.tmpl'], 8: ['docs/x.md'] }; + const body = buildReport(args, [run1()], merges, deps, m => diff(files[m.pr]!, { pr: m.pr, sha: m.sha })); + const report: ReplayReport = { repo: 'o/r', ref: 'origin/main', head: '0123456789', ...body }; + const text = formatReport(report); + const screenLines = text.split('\n').filter(line => /\bscreen\b/i.test(line)); + expect(screenLines.length).toBeGreaterThan(3); + for (const line of screenLines) expect(line).toContain(SCREEN_LABEL); + expect(text).toContain(`${SCREEN_LABEL}: its preamble detector is a line-commonality heuristic`); + expect(body.model!.rows).toHaveLength(12); + expect(JSON.stringify(body.prs[0])).toContain('"screenProvisional"'); + }); +});