From 16f5d2f0a4e55459ac3b964942d0de3b4ae8457e Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 2 Nov 2026 10:31:33 +0000 Subject: [PATCH] feat(test-impact): main receipts and shadow mode for free tests (GST-167 PR 3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .gstack/pregate.json | 3 + .../capy-test-impact-receipts-shadow.md | 1 + docs/test-impact.md | 39 ++- lib/run-evidence.ts | 4 +- lib/test-impact/key.ts | 4 +- scripts/lib/free-receipts.ts | 257 ++++++++++++++++++ scripts/test-free-shards.ts | 31 ++- test/fixtures/module-size-ratchet.json | 1 + test/fixtures/shard-cli-parity/results.json | 66 ++++- test/free-receipts.test.ts | 225 +++++++++++++++ 10 files changed, 611 insertions(+), 20 deletions(-) create mode 100644 changelog.d/capy-test-impact-receipts-shadow.md create mode 100644 scripts/lib/free-receipts.ts create mode 100644 test/free-receipts.test.ts diff --git a/.gstack/pregate.json b/.gstack/pregate.json index 0454e74f3..4360ae865 100644 --- a/.gstack/pregate.json +++ b/.gstack/pregate.json @@ -20,5 +20,8 @@ "windows-free-tests.yml/windows-free-shard": "bun run scripts/test-free-shards.ts --windows-only --list" }, "warn": [], + "run_profiles": { + "free-receipts": { "workflow": "free-tests.yml", "events": ["push"], "jobs": ["free-tests"], "artifact": "free-tests-tree", "max_age_hours": 72 } + }, "strays": { "allow": ["modules/loadouts.json", "modules/loadouts.schema.json"] } } diff --git a/changelog.d/capy-test-impact-receipts-shadow.md b/changelog.d/capy-test-impact-receipts-shadow.md new file mode 100644 index 000000000..2f9cf8b73 --- /dev/null +++ b/changelog.d/capy-test-impact-receipts-shadow.md @@ -0,0 +1 @@ +- **Test impact: main receipts and shadow mode (GST-167, part 3).** A free-tests shard on a main push can now record, with `--receipts`, which files passed on the first try and under which key. New `--shadow-select` checks those receipts against GitHub (same repository, push event, first attempt, recent, the right workflow, a head on main, and the tree and commit the run really tested), computes each pull request file's key with main's key code, and prints one line per free file: `would-reuse key=…` or `execute ()`, plus the share of files and seed time it could have skipped. Anything it can't verify reads `execute (receipts unavailable: …)`. It runs nothing differently; every file still runs. See docs/test-impact.md. diff --git a/docs/test-impact.md b/docs/test-impact.md index 418ca07d8..962f9487c 100644 --- a/docs/test-impact.md +++ b/docs/test-impact.md @@ -24,7 +24,7 @@ The recorder runs `strace -f -y -qq -v` with `openat, open, execve, execveat, ge - the names in each traced directory listing, at the tree; - the test's forward import-graph closure from `lib/pregate/import-graph.ts`, up to 12 hops; - `bun.lock`, `bunfig.toml`, every `tsconfig*.json` and the `dependencies` in `.gstack/pregate.json`; -- the key code itself: the tracer, the key, the CLI, the free-test runner and planner (`scripts/test-free-shards.ts`) with every repo module it imports (shard engine, HOME guard, CI health, Windows curation, the paid/free split, the state root), the import graph and `lib/pregate/lane-selection.ts`; +- the key code itself: the tracer, the key, the CLI, the free-test runner and planner (`scripts/test-free-shards.ts`) with every repo module it imports (shard engine, HOME guard, CI health, Windows curation, the paid/free split, the state root), the import graph, `lib/pregate/lane-selection.ts`, and the receipt reader (`scripts/lib/free-receipts.ts`) with the run verifier it trusts (`lib/run-evidence.ts`); - the bytes of each executable outside the repo the trace ran (git, bash, tar and so on), read on the machine computing the key; - the runtime (platform, architecture, Bun version) and the values of `CI`, `GITHUB_ACTIONS`, `TZ`, `LANG`, `LC_ALL`, `NODE_ENV`, `EVALS` and `EVALS_TIER`. @@ -39,4 +39,39 @@ A file is ineligible, so it always runs, when: - its import-graph walk reaches the 12-hop cap. The walk fails closed: a deeper chain makes the test ineligible instead of cutting the closure short; - the tree changes any key-code file against the base. Then every file runs. -Limits: strace can't see which environment variables a test reads, so only the allowlist above is in the key, and a test that depends on another variable has to read it through a file the trace sees or stay off the reuse path. The recorder reuses the runner's arguments and sandbox but not its home-write guard, which only watches and changes nothing a test reads. Timing assertions, flaky history and the shard's execution grouping are for PR 3 and PR 4, which have the receipts to check them against. Windows has no tracer, so the Windows lane stays full. +Limits: strace can't see which environment variables a test reads, so only the allowlist above is in the key, and a test that depends on another variable has to read it through a file the trace sees or stay off the reuse path. The recorder reuses the runner's arguments and sandbox but not its home-write guard, which only watches and changes nothing a test reads. Timing assertions, flaky history across the last main runs and the shard's execution grouping are for PR 4, which reads them from the receipts PR 3 writes. Windows has no tracer, so the Windows lane stays full. + +## Receipts and shadow mode + +PR 3 records what main passed and reports, on every pull request, what the free lane could have reused. It still runs every file. `scripts/lib/free-receipts.ts` holds the logic; `scripts/test-free-shards.ts` has two thin hooks. + +**Writing receipts.** `--ci-run --shard --result --receipts [--traces ]` writes `/free-receipts-shard-.json` after the shard runs. Each passing file whose key exists gets one record: `file`, `key` (the full sha256), `run_id`, `run_attempt`, `head_sha`, `tree`, `shard` and `written_at`. The key is computed at the run's own commit with that commit's key code (through the CLI's `key --base`), from the traces in `--traces` (default `.context/test-impact/traces`). A shard writes nothing at all when: + +- it didn't pass, its flaky retry re-ran anything, a worker crashed (even if bun's own retry then passed), it recorded an unattributed failure, or its summary doesn't count every planned file; +- the run isn't a `push` run, isn't on its first attempt, or the checkout isn't the run's commit. + +A failure while writing prints `none written ()` and never changes the shard's verdict. The Windows lane has no tracer, so `--receipts` refuses `--windows-only`. + +**Reading receipts.** `--shadow-select --receipts --base [--traces ]` reads every receipt file under `` and checks each source run once through `verifyRun` (see [run evidence](pregate.md#run-evidence)). It uses the base branch's `run_profiles["free-receipts"]` from `.gstack/pregate.json`, so a pull request can't relax it, and it accepts `push` runs only, whatever the profile's events say. The profile names the `free-tests` job, which passes only when every shard did, and the `free-tests-tree` artifact, which holds the run's `tree.json`; receipts and the run may be at most 72 hours old. A receipt is refused, with its reason on the file's line, when: + +- `other_workflow`: its run executed another workflow file; +- `not_push`: its run was a `pull_request`, a dispatch or any other event; +- `retried`: the receipt or the API names a run attempt above 1, or the run moved to a new attempt while it was being checked; +- `stale`: the run finished, or the receipt was written, more than the profile's `max_age_hours` ago; +- `forged_run`: the receipt's sha or tree disagrees with what the API and git say the run tested, `tree.json` disagrees with git, or `tree.json` doesn't list the file; +- `off_base`: the run's head isn't an ancestor of `--base` (so the PR job needs main's history); +- `run_refused`: any other `verifyRun` refusal, such as a failed run or a newer red run on the same head. + +The PR's keys are computed at `HEAD` by the base's key code: the reader runs `scripts/test-impact.ts key --tree HEAD --base --json`, the trusted route above, and a failure there makes every line `execute (keys unavailable: )`. The writer runs the same command with the run's commit as both tree and base. + +**The report.** One line per free file, then a summary: + +``` +test/a.test.ts would-reuse 19283746501 key=1f2e3d4c5b6a +test/b.test.ts execute (key mismatch: receipt 9a8b7c6d5e4f, tree 0a1b2c3d4e5f) +test/c.test.ts execute (ineligible: no trace) +test/d.test.ts execute (no receipt) +shadow: would reuse 1 of 4 free files (25.0%) and 3.0 s of 10.0 s seed time (30.0%); 0 files have no seed duration. Nothing was skipped: every file ran. +``` + +The seed time comes from `scripts/free-test-durations.json`. The report throws unless its lines cover every free file exactly once. It never fails open: if the receipts can't be read or verified (a missing directory, a malformed file, no profile on the base, an API or artifact outage), every line reads `execute (receipts unavailable: )`. It exits 0 and runs nothing, so CI wiring keeps it non-blocking. The reader runs from the PR's checkout for now; that's fine while it only reports, and PR 4 moves the decision to the base's code before anything is skipped. diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index e0f17f437..e99d731e4 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -227,9 +227,9 @@ function gh(args: string[], timeoutMs = 60_000): string { /** The gh- and git-backed dependencies for `repo`, reading trees in `repoRoot` (fetching a missing head from origin once). */ export function ghRunDeps(repo: string, repoRoot: string): RunDeps { - const json = (p: string): T => JSON.parse(gh(['api', p])); + const json = (p: string): T => JSON.parse(gh(['api', p])); // Every page, one JSON value per line: a veto or a required job past the first 100 entries must still be seen. - const all = (p: string, key: string): T[] => gh(['api', '--paginate', p, '--jq', `.${key}[]`]).split('\n').filter(Boolean).map(l => JSON.parse(l) as T); + const all = (p: string, key: string): T[] => gh(['api', '--paginate', p, '--jq', `.${key}[]`]).split('\n').filter(Boolean).map(l => JSON.parse(l) as T); return { run: id => json(`repos/${repo}/actions/runs/${id}`), runsForHead: (workflow, sha) => all(`repos/${repo}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100`, 'workflow_runs'), diff --git a/lib/test-impact/key.ts b/lib/test-impact/key.ts index 76ec75196..e60450ef0 100644 --- a/lib/test-impact/key.ts +++ b/lib/test-impact/key.ts @@ -38,12 +38,12 @@ export const MAX_HOPS = 12; export const TRACE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; /** How far in the future a trace's timestamp may sit (clock skew) before it counts as stale. */ export const TRACE_CLOCK_SKEW_MS = 5 * 60 * 1000; -/** Tracer, key, CLI, the runner/planner and every repo module it imports, and the graph and always-run code the key reads. PR 4 adds its verifier here. */ +/** Tracer, key, CLI, the runner/planner and every repo module it imports, the graph and always-run code the key reads, and the receipt reader with the run verifier it trusts. PR 4 adds its verifier here. */ export const KEY_CODE_FILES = [ 'lib/test-impact/trace.ts', 'lib/test-impact/key.ts', 'scripts/test-impact.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/test-selection.ts', 'lib/state-root.ts', - 'lib/pregate/import-graph.ts', 'lib/pregate/lane-selection.ts', + 'lib/pregate/import-graph.ts', 'lib/pregate/lane-selection.ts', 'scripts/lib/free-receipts.ts', 'lib/run-evidence.ts', ]; const TSCONFIG = /(?:^|\/)tsconfig[^/]*\.json$/; diff --git a/scripts/lib/free-receipts.ts b/scripts/lib/free-receipts.ts new file mode 100644 index 000000000..0d9076ea5 --- /dev/null +++ b/scripts/lib/free-receipts.ts @@ -0,0 +1,257 @@ +/** + * free-receipts — main receipts for free test files and the pull request shadow report (GST-167 PR 3; + * docs/test-impact.md#receipts-and-shadow-mode). Nothing here changes what CI runs. + * + * Writing (scripts/test-free-shards.ts --ci-run … --receipts ): a shard of a free-tests.yml main push run + * that passed on its first attempt, with no flaky retry, crashed worker, unattributed failure or short summary, + * writes one record per passing file whose key lib/test-impact/key.ts computes at the run's tree with that tree's + * own key code. Any doubt writes nothing for the whole shard; a write failure never changes the shard's verdict. + * + * Reading (--shadow-select --receipts --base ): each source run is checked once through + * lib/run-evidence.ts verifyRun under the base branch's run_profiles["free-receipts"], push events only. A record is + * refused, with a named reason, when its run is another workflow (other_workflow), not a push (not_push), retried + * (retried: run_attempt above 1 on the receipt or the API, or the run moved meanwhile), older than the profile's + * max_age_hours (stale), disagrees with the API or git on sha, tree or the files the run executed (forged_run), + * ran on a head that is not on the base (off_base), or fails any other verifyRun check (run_refused). An API or + * artifact outage makes every receipt unavailable. A file is reported would-reuse only when an accepted receipt's + * key equals its key at the PR tree computed by the base's key code; every other file reports execute (). + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { bunIsolationArgs } from '../../lib/bun-isolation'; +import { PREGATE_CONFIG_PATH } from '../../lib/pregate/config'; +import { DEFAULT_MAX_AGE_HOURS, ghRunDeps, runProfileErrors, verifyRun, type RunDeps, type RunEvidence, type RunProfile, type RunRefusal, type RunVerdict } from '../../lib/run-evidence'; +import type { KeyReport } from '../../lib/test-impact/key'; +import type { FreeShardOutcome } from '../test-free-shards'; + +export const RECEIPTS_PROFILE = 'free-receipts'; +export const RECEIPT_VERSION = 1; +const RECEIPT_FILE = /^free-receipts-shard-(\d+)\.json$/; +const SHA = /^[0-9a-f]{40}$/; +const TREE = /^[0-9a-f]{40,64}$/; +const KEY = /^[0-9a-f]{64}$/; +const CLOCK_SKEW_MS = 5 * 60 * 1000; + +export interface FreeReceipt { file: string; key: string; run_id: number; run_attempt: number; head_sha: string; tree: string; shard: number; written_at: string } +export type ReceiptRefusal = 'forged_run' | 'not_push' | 'retried' | 'stale' | 'other_workflow' | 'off_base' | 'run_refused'; +export interface ReceiptCheck { repo: string; profileName: string; profile: RunProfile; now: Date; deps: RunDeps; onBase: (sha: string) => boolean } +export type ReceiptVerdict = { receipt: FreeReceipt; ok: true; evidence: RunEvidence } | { receipt: FreeReceipt; ok: false; reason: ReceiptRefusal; detail: string }; +export type ReceiptVerdicts = { verdicts: ReceiptVerdict[] } | { unavailable: string }; +/** Keys for `tests` at `tree`, computed by `base`'s own key code. */ +export type KeysAt = (tree: string, base: string, tests: string[]) => Promise; +type Env = Record; + +const short = (s: string) => s.slice(0, 12); +const message = (e: unknown) => String((e as Error)?.message ?? e).slice(0, 300); + +/** + * scripts/test-impact.ts's trusted route, `key --base`: the key code is loaded from `base`, never from this checkout. + * It runs as a subprocess so the runner neither imports the CLI (which imports the runner) nor its computed import. + */ +function trustedKeysAt(root: string, traceDir?: string): KeysAt { + return async (tree, base, tests) => { + const args = [...bunIsolationArgs(), path.join(root, 'scripts', 'test-impact.ts'), 'key', '--tree', tree, '--base', base, '--json', ...(traceDir ? ['--traces', traceDir] : []), ...tests]; + const r = spawnSync(process.execPath, args, { cwd: root, encoding: 'utf8', timeout: 1_800_000, maxBuffer: 1024 * 1024 * 1024 }); + if (r.status !== 0) throw new Error(`test-impact key exited ${r.status ?? r.signal ?? r.error?.message}: ${(r.stderr ?? '').trim().slice(-300)}`); + return JSON.parse(r.stdout) as KeyReport; + }; +} + +function git(root: string, args: string[]): { status: number | null; stdout: string } { + const r = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); + return { status: r.error ? null : r.status, stdout: r.stdout ?? '' }; +} + +function shardVeto(outcome: FreeShardOutcome, retry: FreeShardOutcome | null): string | null { + if (retry) return 'the flaky retry re-ran files in this shard'; + if (outcome.status !== 'passed' || outcome.exitCode !== 0) return `the shard ${outcome.status} (exit ${outcome.exitCode})`; + if (outcome.failingFiles.length || outcome.unattributedFailures) return 'the shard recorded failures'; + if (outcome.crashedFiles?.length) return `a worker crashed on ${outcome.crashedFiles.join(', ')}`; + const s = outcome.summary; + if (!outcome.files.length || !s?.sawTerminalSummary || s.filesRan !== outcome.files.length) return 'the shard has no complete execution summary'; + return null; +} + +function runVeto(env: Env, revision: string): string | null { + if (env.GITHUB_EVENT_NAME !== 'push') return `a ${env.GITHUB_EVENT_NAME || 'local'} run writes no receipts; only main push runs do`; + if (!/^\d{1,15}$/.test(env.GITHUB_RUN_ID ?? '')) return 'GITHUB_RUN_ID is not a run id'; + if (env.GITHUB_RUN_ATTEMPT !== '1') return `run attempt ${env.GITHUB_RUN_ATTEMPT ?? 'unknown'} writes no receipts; first attempts only`; + if (env.GITHUB_SHA !== revision) return `the checkout ${short(revision)} is not the run's commit ${short(env.GITHUB_SHA ?? 'unknown')}`; + return null; +} + +/** Write shard `shard`'s receipts into `dir` and return the line to print. Never throws: a failure writes nothing. */ +export async function recordShardReceipts(o: { + root: string; dir: string; shard: number; outcome: FreeShardOutcome; retry: FreeShardOutcome | null; revision: string; + traceDir?: string; env?: Env; now?: Date; treeOf?: (sha: string) => string; keysAt?: KeysAt; +}): Promise { + const label = `[test:free] receipts for shard ${o.shard}:`; + try { + const env = o.env ?? process.env; + const veto = shardVeto(o.outcome, o.retry) ?? runVeto(env, o.revision); + if (veto) return `${label} none written (${veto})`; + const tree = (o.treeOf ?? (sha => git(o.root, ['rev-parse', '--verify', `${sha}^{tree}`]).stdout.trim()))(o.revision); + if (!TREE.test(tree)) throw new Error(`git cannot read the tree of ${o.revision}`); + const report = await (o.keysAt ?? trustedKeysAt(o.root, o.traceDir))(o.revision, o.revision, o.outcome.files); + const keyed = new Map(report.results.filter(r => r.eligible && r.key && KEY.test(r.key)).map(r => [r.test, r.key!])); + const written_at = (o.now ?? new Date()).toISOString(); + const receipts: FreeReceipt[] = o.outcome.files.filter(file => keyed.has(file)).map(file => ({ + file, key: keyed.get(file)!, run_id: Number(env.GITHUB_RUN_ID), run_attempt: 1, head_sha: o.revision, tree, shard: o.shard, written_at, + })); + const unkeyed = o.outcome.files.length - receipts.length; + if (!receipts.length) return `${label} none written (no file has a key: ${unkeyed} ineligible)`; + const file = path.join(o.dir, `free-receipts-shard-${o.shard}.json`); + fs.mkdirSync(o.dir, { recursive: true }); + fs.writeFileSync(`${file}.tmp-${process.pid}`, JSON.stringify({ version: RECEIPT_VERSION, receipts }, null, 2) + '\n'); + fs.renameSync(`${file}.tmp-${process.pid}`, file); + return `${label} ${receipts.length} written, ${unkeyed} without a key (ineligible) → ${file}`; + } catch (e) { + return `${label} none written (${message(e)})`; + } +} + +function asReceipt(r: unknown): FreeReceipt | null { + const x = r as Record; + if (!x || typeof x !== 'object') return null; + const posInt = (v: unknown) => Number.isInteger(v) && (v as number) > 0; + const ok = typeof x.file === 'string' && /^[\w@.\/-]+$/.test(x.file) && !x.file.split('/').includes('..') && !x.file.startsWith('/') + && typeof x.key === 'string' && KEY.test(x.key) && posInt(x.run_id) && posInt(x.run_attempt) && posInt(x.shard) + && typeof x.head_sha === 'string' && SHA.test(x.head_sha) && typeof x.tree === 'string' && TREE.test(x.tree) + && typeof x.written_at === 'string' && Number.isFinite(Date.parse(x.written_at)); + if (!ok) return null; + const { file, key, run_id, run_attempt, head_sha, tree, shard, written_at } = x as unknown as FreeReceipt; + return { file, key, run_id, run_attempt, head_sha, tree, shard, written_at }; +} + +/** Every receipt file under `dir` (artifact downloads may nest one directory per shard), in shard order. */ +export function readReceipts(dir: string): { receipts: FreeReceipt[] } | { unavailable: string } { + if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) return { unavailable: `no receipts directory at ${dir}` }; + const shardOf = (f: string) => Number(RECEIPT_FILE.exec(path.basename(f))?.[1]); + const files = (fs.readdirSync(dir, { recursive: true }) as string[]).filter(f => RECEIPT_FILE.test(path.basename(f))) + .sort((a, b) => shardOf(a) - shardOf(b) || a.localeCompare(b)); + if (!files.length) return { unavailable: `no receipt files in ${dir}` }; + const receipts: FreeReceipt[] = []; + for (const f of files) { + let body: { version?: unknown; receipts?: unknown }; + try { body = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')); } catch (e) { return { unavailable: `malformed ${f}: ${message(e)}` }; } + if (body?.version !== RECEIPT_VERSION || !Array.isArray(body.receipts)) return { unavailable: `malformed ${f}: not a version ${RECEIPT_VERSION} receipts file` }; + for (const [i, r] of body.receipts.entries()) { + const receipt = asReceipt(r); + if (!receipt) return { unavailable: `malformed ${f}: record ${i} is not a receipt` }; + receipts.push(receipt); + } + } + return { receipts }; +} + +const RUN_REFUSALS: Partial> = { + workflow_mismatch: 'other_workflow', event_mismatch: 'not_push', run_stale: 'stale', tree_mismatch: 'forged_run', artifact_mismatch: 'forged_run', run_changed: 'retried', +}; +const RUN_UNAVAILABLE: RunRefusal[] = ['api_error', 'artifact_missing']; + +/** Each receipt's verdict, its run verified once; an API or artifact outage makes them all unavailable. */ +export function verifyReceipts(receipts: FreeReceipt[], c: ReceiptCheck): ReceiptVerdicts { + const profile: RunProfile = { ...c.profile, events: ['push'] }; + const maxAgeHours = profile.max_age_hours ?? DEFAULT_MAX_AGE_HOURS; + const runs = new Map(); + for (const id of new Set(receipts.map(r => r.run_id))) { + const v = verifyRun(c.deps, { repo: c.repo, runRef: String(id), profileName: c.profileName, profile, now: c.now }); + if (!v.ok && RUN_UNAVAILABLE.includes(v.reason)) return { unavailable: `run ${id} ${v.reason}: ${v.detail}` }; + runs.set(id, v); + } + const onBase = new Map(); + return { + verdicts: receipts.map((receipt): ReceiptVerdict => { + const refuse = (reason: ReceiptRefusal, detail: string): ReceiptVerdict => ({ receipt, ok: false, reason, detail }); + const v = runs.get(receipt.run_id)!; + if (!v.ok) return refuse(RUN_REFUSALS[v.reason] ?? 'run_refused', `${v.reason}: ${v.detail}`); + const e = v.evidence; + if (e.run_attempt !== 1 || receipt.run_attempt !== 1) return refuse('retried', `run ${e.run_id} is on attempt ${e.run_attempt} and the receipt names attempt ${receipt.run_attempt}; only first attempts count`); + if (receipt.head_sha !== e.head_sha || receipt.tree !== e.tree) return refuse('forged_run', `the receipt names ${short(receipt.head_sha)} tree ${short(receipt.tree)}, but run ${e.run_id} tested ${short(e.head_sha)} tree ${short(e.tree)}`); + if (!e.files.includes(receipt.file)) return refuse('forged_run', `run ${e.run_id}'s tree.json does not list ${receipt.file}`); + const age = c.now.getTime() - Date.parse(receipt.written_at); + if (!(age >= -CLOCK_SKEW_MS && age <= maxAgeHours * 3600_000)) return refuse('stale', `written ${receipt.written_at}, limit ${maxAgeHours} h`); + if (!onBase.has(e.head_sha)) onBase.set(e.head_sha, c.onBase(e.head_sha)); + if (!onBase.get(e.head_sha)) return refuse('off_base', `run ${e.run_id}'s head ${short(e.head_sha)} is not on the base branch`); + return { receipt, ok: true, evidence: e }; + }), + }; +} + +/** One line per free file and the reusable share. Throws unless the lines cover `files` exactly once. */ +export function shadowReport(o: { files: string[]; receipts: ReceiptVerdicts; keys: KeyReport | { unavailable: string }; durations: Record }): { lines: string[]; summary: string; reusable: string[] } { + const keys = 'unavailable' in o.keys ? null : new Map(o.keys.results.map(r => [r.test, r])); + const byFile = new Map(); + if ('verdicts' in o.receipts) { + for (const v of [...o.receipts.verdicts].sort((a, b) => b.receipt.written_at.localeCompare(a.receipt.written_at))) byFile.set(v.receipt.file, [...(byFile.get(v.receipt.file) ?? []), v]); + } + const entries = o.files.map(file => { + const execute = (reason: string) => ({ file, reuse: false, line: `${file} execute (${reason})` }); + if ('unavailable' in o.receipts) return execute(`receipts unavailable: ${o.receipts.unavailable}`); + if (!keys) return execute(`keys unavailable: ${(o.keys as { unavailable: string }).unavailable}`); + const k = keys.get(file); + if (!k) return execute('no key computed'); + if (!k.eligible || !k.key) return execute(`ineligible: ${k.reasons[0] ?? 'no key'}${k.reasons.length > 1 ? ` (+${k.reasons.length - 1} more)` : ''}`); + const candidates = byFile.get(file) ?? []; + if (!candidates.length) return execute('no receipt'); + const match = candidates.find(v => v.ok && v.receipt.key === k.key); + if (match) return { file, reuse: true, line: `${file} would-reuse ${match.receipt.run_id} key=${short(k.key)}` }; + const accepted = candidates.find(v => v.ok); + if (accepted) return execute(`key mismatch: receipt ${short(accepted.receipt.key)}, tree ${short(k.key)}`); + const refused = candidates[0] as Extract; + return execute(`${refused.reason}: ${refused.detail}`); + }); + const covered = new Set(entries.map(e => e.file)); + if (entries.length !== covered.size || covered.size !== new Set(o.files).size || o.files.some(f => !covered.has(f))) { + throw new Error(`shadow report must cover every free file exactly once: ${entries.length} lines for ${covered.size} distinct files`); + } + const reusable = entries.filter(e => e.reuse).map(e => e.file); + const ms = (files: string[]) => files.reduce((sum, f) => sum + (o.durations[f] ?? 0), 0); + const pct = (a: number, b: number) => `${b ? ((100 * a) / b).toFixed(1) : '0.0'}%`; + const secs = (n: number) => `${(n / 1000).toFixed(1)} s`; + const unseeded = o.files.filter(f => o.durations[f] === undefined).length; + const summary = `shadow: would reuse ${reusable.length} of ${o.files.length} free files (${pct(reusable.length, o.files.length)}) and ${secs(ms(reusable))} of ${secs(ms(o.files))} seed time (${pct(ms(reusable), ms(o.files))}); ${unseeded} files have no seed duration. Nothing was skipped: every file ran.`; + return { lines: entries.map(e => e.line), summary, reusable }; +} + +/** The base branch's run_profiles["free-receipts"], so a pull request can't relax the rules its receipts are read under. */ +function receiptProfile(root: string, base: string): RunProfile | { unavailable: string } { + if (base.startsWith('-')) return { unavailable: `${base} is not a ref` }; + const r = git(root, ['show', `${base}:${PREGATE_CONFIG_PATH}`]); + if (r.status !== 0) return { unavailable: `cannot read ${base}:${PREGATE_CONFIG_PATH}` }; + let profile: unknown; + try { profile = JSON.parse(r.stdout)?.run_profiles?.[RECEIPTS_PROFILE]; } catch (e) { return { unavailable: `${base}:${PREGATE_CONFIG_PATH} is not JSON: ${message(e)}` }; } + if (!profile) return { unavailable: `${base}:${PREGATE_CONFIG_PATH} declares no run_profiles.${RECEIPTS_PROFILE}` }; + const errors = runProfileErrors({ [RECEIPTS_PROFILE]: profile }, undefined); + return errors.length ? { unavailable: `${base}'s run_profiles.${RECEIPTS_PROFILE} is invalid: ${errors[0]!.message}` } : profile as RunProfile; +} + +function repoName(root: string, env: Env): string | null { + if (/^[\w.-]+\/[\w.-]+$/.test(env.GITHUB_REPOSITORY ?? '')) return env.GITHUB_REPOSITORY!; + return /github\.com[:/]([\w.-]+\/[\w.-]+?)(?:\.git)?$/.exec(git(root, ['remote', 'get-url', 'origin']).stdout.trim())?.[1] ?? null; +} + +/** --shadow-select: verify the receipts in `receiptsDir`, key `files` at HEAD with `base`'s key code, and report. */ +export async function runShadowSelect(o: { + root: string; files: string[]; receiptsDir: string; base: string; durations: Record; + traceDir?: string; now?: Date; env?: Env; keysAt?: KeysAt; deps?: RunDeps; onBase?: (sha: string) => boolean; +}): Promise<{ lines: string[]; summary: string; reusable: string[] }> { + const env = o.env ?? process.env; + const read = readReceipts(o.receiptsDir); + let receipts: ReceiptVerdicts = read as { unavailable: string }; + if ('receipts' in read) { + const profile = receiptProfile(o.root, o.base); + const repo = repoName(o.root, env); + if ('unavailable' in profile) receipts = profile; + else if (!repo) receipts = { unavailable: 'cannot tell which GitHub repository this is (no GITHUB_REPOSITORY, no github.com origin)' }; + else { + const onBase = o.onBase ?? (sha => git(o.root, ['merge-base', '--is-ancestor', sha, o.base]).status === 0); + try { receipts = verifyReceipts(read.receipts, { repo, profileName: RECEIPTS_PROFILE, profile, now: o.now ?? new Date(), deps: o.deps ?? ghRunDeps(repo, o.root), onBase }); } + catch (e) { receipts = { unavailable: message(e) }; } + } + } + const keys = 'unavailable' in receipts ? { unavailable: 'not computed' } + : await (o.keysAt ?? trustedKeysAt(o.root, o.traceDir))('HEAD', o.base, o.files).catch(e => ({ unavailable: message(e) })); + return shadowReport({ files: o.files, receipts, keys, durations: o.durations }); +} diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index ba82c3a48..e2f1336dd 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -78,6 +78,8 @@ * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json * bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/ + * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result r.json --receipts out/ [--traces dir] # main push: write receipts + * bun run scripts/test-free-shards.ts --shadow-select --receipts in/ --base origin/main [--traces dir] # report reuse, run nothing */ import * as fs from 'fs'; @@ -542,6 +544,11 @@ type CliOptions = { result: string | null; results: string | null; filesFrom: string | null; + /** GST-167 receipts: written by --ci-run, read by --shadow-select (scripts/lib/free-receipts.ts). */ + shadowSelect: boolean; + receipts: string | null; + traces: string | null; + base: string | null; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -556,8 +563,9 @@ export function parseCliOptions(argv: string[]): CliOptions { let wallTimeoutExplicit = false; let quick = false; let attributeHome = false; - const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { - ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, + let shadowSelect = false; + const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom' | 'receipts' | 'traces' | 'base', string | null> = { + ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, receipts: null, traces: null, base: null, }; const pathFlag = (flag: string, key: keyof typeof paths) => (next: () => string | undefined) => { const value = next(); @@ -579,6 +587,10 @@ export function parseCliOptions(argv: string[]): CliOptions { '--result': pathFlag('--result', 'result'), '--results': pathFlag('--results', 'results'), '--files-from': pathFlag('--files-from', 'filesFrom'), + '--shadow-select': () => { shadowSelect = true; }, + '--receipts': pathFlag('--receipts', 'receipts'), + '--traces': pathFlag('--traces', 'traces'), + '--base': pathFlag('--base', 'base'), '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -603,7 +615,9 @@ export function parseCliOptions(argv: string[]): CliOptions { if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths }; + if ((paths.receipts && !paths.ciRun && !shadowSelect) || (paths.traces && !paths.receipts) || (paths.receipts && windowsOnly)) throw new Error('--receipts (and --traces) ride a Linux --ci-run, which writes receipts, or --shadow-select, which reads them'); + if (shadowSelect !== Boolean(paths.base) || (shadowSelect && (!paths.receipts || ciModes || quick || listOnly || dryRun || recordDurations || attributeHome || paths.filesFrom))) throw new Error('--shadow-select needs --receipts and --base and takes no other mode'); + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, shadowSelect, ...paths }; } const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; @@ -1075,6 +1089,8 @@ export interface FreeShardOutcome { unattributedFailures: number; /** Named failures with their excerpts, so a flaky pass records what failed the first time. */ failures?: FreeRunFailure[]; + /** Files that crashed a worker, even when bun's own retry then passed; any entry vetoes the shard's receipts. */ + crashedFiles?: string[]; } export interface ShardCommand { @@ -1744,6 +1760,7 @@ export async function runFreeShard( const outcome: FreeShardOutcome = { shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures, ...(status === 'passed' ? {} : { failures: report.failures }), + ...(report.crashedFiles.length ? { crashedFiles: report.crashedFiles } : {}), summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary }, }; log(shardEpilogue(outcome, totalShards)); @@ -1951,9 +1968,17 @@ async function main(): Promise { }); const retried = await retryFailedFreeFiles([outcome], plan.shards.length, options); writeJson(options.result!, { planId: plan.id, revision, outcome, retry: retried.retry } satisfies FreeCiResult); + if (options.receipts) console.log(await (await import('./lib/free-receipts')).recordShardReceipts({ root: ROOT, dir: options.receipts, shard: shard.shard, outcome, retry: retried.retry, revision, traceDir: options.traces ?? undefined })); return retried.exitCode; } + if (options.shadowSelect) { + const report = await (await import('./lib/free-receipts')).runShadowSelect({ root: ROOT, files: allFiles, receiptsDir: options.receipts!, base: options.base!, traceDir: options.traces ?? undefined, durations: loadFreeTestDurations() ?? {} }); + for (const line of [...report.lines, report.summary]) console.log(line); + appendStepSummary(report.summary); + return 0; + } + let files = allFiles; if (options.filesFrom) { files = readFilesFrom(options.filesFrom, allFiles); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index 6d0298710..d9bafa4eb 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -59,6 +59,7 @@ { "file": "scripts/eval-bg.ts", "movedFrom": "(new: eval:bg dispatch/local backend, W5d)" }, { "file": "test/helpers/module-size.ts", "movedFrom": "(new: ratchet (c) counter)" }, { "file": "scripts/lib/free-home-guard.ts", "movedFrom": "(new: free-lane home-write tripwire, kept out of scripts/test-free-shards.ts)" }, + { "file": "scripts/lib/free-receipts.ts", "movedFrom": "(new: GST-167 main receipts and shadow report, kept out of scripts/test-free-shards.ts)" }, { "file": "browse/src/routes/activity.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/commands.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/core.ts", "movedFrom": "browse/src/server.ts" }, diff --git a/test/fixtures/shard-cli-parity/results.json b/test/fixtures/shard-cli-parity/results.json index cb5bc092d..abc166dd2 100644 --- a/test/fixtures/shard-cli-parity/results.json +++ b/test/fixtures/shard-cli-parity/results.json @@ -21,7 +21,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -49,7 +53,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -76,7 +84,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -102,7 +114,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -128,7 +144,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -157,7 +177,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -184,7 +208,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -213,7 +241,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -244,7 +276,11 @@ "ciRun": "plan.json", "ciVerify": null, "result": "result.json", - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -273,7 +309,11 @@ "ciRun": null, "ciVerify": "plan.json", "result": null, - "results": "results" + "results": "results", + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, @@ -424,7 +464,11 @@ "ciRun": null, "ciVerify": null, "result": null, - "results": null + "results": null, + "shadowSelect": false, + "receipts": null, + "traces": null, + "base": null } } }, diff --git a/test/free-receipts.test.ts b/test/free-receipts.test.ts new file mode 100644 index 000000000..2af4f1d73 --- /dev/null +++ b/test/free-receipts.test.ts @@ -0,0 +1,225 @@ +/** + * GST-167 PR 3: scripts/lib/free-receipts.ts writes main receipts from a clean first-attempt shard and reads them back + * for a pull request's shadow report through lib/run-evidence.ts verifyRun. Every refusal has a fixture; RunDeps and + * the key computation are fakes, so nothing here reaches GitHub or runs a test. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import type { ApiJob, ApiRun, RunDeps, RunProfile, TreeRecord } from '../lib/run-evidence'; +import type { KeyReport } from '../lib/test-impact/key'; +import { readReceipts, recordShardReceipts, runShadowSelect, shadowReport, verifyReceipts, type FreeReceipt, type ReceiptCheck } from '../scripts/lib/free-receipts'; +import { parseCliOptions, type FreeShardOutcome } from '../scripts/test-free-shards'; + +const REPO = 'garrytan/gstack'; +const HEAD = 'a'.repeat(40); +const TREE = 'b'.repeat(40); +const NOW = new Date('2026-11-02T12:00:00Z'); +const PROFILE: RunProfile = { workflow: 'free-tests.yml', events: ['push'], jobs: ['free-tests'], artifact: 'free-tests-tree', max_age_hours: 72 }; +const A = 'test/a.test.ts'; +const B = 'test/b.test.ts'; +const C = 'test/c.test.ts'; +const D = 'test/d.test.ts'; +const KEYS: Record = { [A]: '1'.repeat(64), [B]: '2'.repeat(64), [C]: '3'.repeat(64), [D]: '4'.repeat(64) }; +const dirs: string[] = []; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); +const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-free-receipts-')); dirs.push(d); return d; }; + +interface World { run: ApiRun; jobs: ApiJob[]; artifact: TreeRecord; trees: Record; throwOn?: boolean } +function world(over: { run?: Partial; artifact?: Partial; jobs?: ApiJob[]; throwOn?: boolean } = {}): World { + const run: ApiRun = { + id: 100, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'push', head_sha: HEAD, + path: '.github/workflows/free-tests.yml', created_at: '2026-11-02T10:00:00Z', updated_at: '2026-11-02T10:20:00Z', + repository: { full_name: REPO }, head_repository: { full_name: REPO }, ...over.run, + }; + return { + run, jobs: over.jobs ?? [{ name: 'free-tests', conclusion: 'success', completed_at: run.updated_at }], + artifact: { run_id: 100, run_attempt: run.run_attempt, event: run.event, sha: HEAD, tree: TREE, files: [A, B, C, D], ...over.artifact }, + trees: { [HEAD]: TREE }, throwOn: over.throwOn, + }; +} +function deps(w: World): RunDeps { + return { + run: id => { if (w.throwOn) throw new Error('HTTP 502'); if (id !== w.run.id) throw new Error('HTTP 404'); return w.run; }, + runsForHead: (_wf, sha) => [w.run].filter(r => r.head_sha === sha), + jobs: () => w.jobs, + artifact: () => w.artifact, + commitTree: sha => w.trees[sha] ?? null, + }; +} +const receipt = (file: string, over: Partial = {}): FreeReceipt => + ({ file, key: KEYS[file]!, run_id: 100, run_attempt: 1, head_sha: HEAD, tree: TREE, shard: 1, written_at: '2026-11-02T10:10:00Z', ...over }); +const check = (w: World, over: Partial = {}): ReceiptCheck => + ({ repo: REPO, profileName: 'free-receipts', profile: PROFILE, now: NOW, deps: deps(w), onBase: () => true, ...over }); +function reasons(receipts: FreeReceipt[], c: ReceiptCheck): string[] { + const v = verifyReceipts(receipts, c); + if ('unavailable' in v) return [`unavailable: ${v.unavailable}`]; + return v.verdicts.map(x => x.ok ? 'ok' : x.reason); +} +const keyReport = (results: Array<{ test: string; key?: string | null; reasons?: string[] }>): KeyReport => ({ + tree: 'HEAD', base: 'origin/main', keyCodeEdited: [], alwaysRunGlobs: { base: [], tree: [] }, + results: results.map(r => ({ test: r.test, eligible: !r.reasons?.length, reasons: r.reasons ?? [], key: r.reasons?.length ? null : (r.key ?? KEYS[r.test]!), inputs: [], tracedAt: null })), +}); + +describe('verifyReceipts: a main receipt stands only on its verified push run', () => { + test('first-attempt receipts from a verified main push run are accepted', () => { + expect(reasons([receipt(A), receipt(B)], check(world()))).toEqual(['ok', 'ok']); + }); + + test('a forged run: the API or git disagrees with the receipt on tree, sha or the files it ran', () => { + expect(reasons([receipt(A, { tree: 'd'.repeat(40) })], check(world()))).toEqual(['forged_run']); + expect(reasons([receipt(A, { head_sha: 'e'.repeat(40) })], check(world()))).toEqual(['forged_run']); + expect(reasons([receipt(A)], check(world({ artifact: { files: [B, C] } })))).toEqual(['forged_run']); + expect(reasons([receipt(A)], check(world({ artifact: { tree: 'f'.repeat(40) } })))).toEqual(['forged_run']); + }); + + test('a pull_request or any other non-push event is refused, even when the profile also accepts dispatches', () => { + expect(reasons([receipt(A)], check(world({ run: { event: 'pull_request' } })))).toEqual(['not_push']); + expect(reasons([receipt(A)], check(world({ run: { event: 'workflow_dispatch' } }), { profile: { ...PROFILE, events: ['push', 'workflow_dispatch'] } }))).toEqual(['not_push']); + }); + + test('a retried run or shard is refused: run_attempt above 1 on the receipt or the API', () => { + expect(reasons([receipt(A, { run_attempt: 2 })], check(world({ run: { run_attempt: 2 } })))).toEqual(['retried']); + expect(reasons([receipt(A)], check(world({ run: { run_attempt: 2 } })))).toEqual(['retried']); + }); + + test('a run or receipt older than the profile max age is stale', () => { + const old = '2026-10-29T10:00:00Z'; + expect(reasons([receipt(A)], check(world({ run: { created_at: old, updated_at: old } })))).toEqual(['stale']); + expect(reasons([receipt(A, { written_at: old })], check(world()))).toEqual(['stale']); + }); + + test('a run of another workflow is refused', () => { + expect(reasons([receipt(A)], check(world({ run: { path: '.github/workflows/evals.yml' } })))).toEqual(['other_workflow']); + }); + + test('a run whose head is not on the base branch is refused', () => { + expect(reasons([receipt(A)], check(world(), { onBase: () => false }))).toEqual(['off_base']); + }); + + test('an API failure makes every receipt unavailable, never accepted', () => { + const v = verifyReceipts([receipt(A), receipt(B)], check(world({ throwOn: true }))); + expect('unavailable' in v && v.unavailable).toContain('HTTP 502'); + }); +}); + +describe('readReceipts', () => { + test('a missing directory, an empty one or a malformed file is unavailable', () => { + expect(readReceipts(path.join(tmp(), 'absent'))).toEqual({ unavailable: expect.stringContaining('no receipts directory') }); + expect(readReceipts(tmp())).toEqual({ unavailable: expect.stringContaining('no receipt files') }); + const bad = tmp(); + fs.writeFileSync(path.join(bad, 'free-receipts-shard-1.json'), JSON.stringify({ version: 1, receipts: [{ file: A, key: 'short' }] })); + expect(readReceipts(bad)).toEqual({ unavailable: expect.stringContaining('malformed') }); + }); + + test('receipt files written by several shards read back together', () => { + const dir = tmp(); + fs.mkdirSync(path.join(dir, 'free-receipts-2')); + fs.writeFileSync(path.join(dir, 'free-receipts-shard-1.json'), JSON.stringify({ version: 1, receipts: [receipt(A)] })); + fs.writeFileSync(path.join(dir, 'free-receipts-2', 'free-receipts-shard-2.json'), JSON.stringify({ version: 1, receipts: [receipt(B, { shard: 2 })] })); + expect(readReceipts(dir)).toEqual({ receipts: [receipt(A), receipt(B, { shard: 2 })] }); + }); +}); + +describe('recordShardReceipts: one record per passing file, first attempts only', () => { + const ENV = { GITHUB_EVENT_NAME: 'push', GITHUB_RUN_ID: '100', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: HEAD }; + const passed = (over: Partial = {}): FreeShardOutcome => ({ + shard: 1, files: [A, B, C], status: 'passed', exitCode: 0, elapsedMs: 10, groupPid: null, failingFiles: [], unattributedFailures: 0, + summary: { testsRan: 9, filesRan: 3, sawTerminalSummary: true }, ...over, + }); + async function record(outcome: FreeShardOutcome, retry: FreeShardOutcome | null = null, env: Record = ENV) { + const dir = tmp(); + let keyed = 0; + const line = await recordShardReceipts({ + root: '/nowhere', dir, shard: 1, outcome, retry, revision: HEAD, env, now: NOW, treeOf: () => TREE, + keysAt: async (tree, base, tests) => { keyed++; expect([tree, base]).toEqual([HEAD, HEAD]); return keyReport(tests.map(t => t === C ? { test: t, reasons: ['no trace'] } : { test: t })); }, + }); + const files = fs.existsSync(dir) ? fs.readdirSync(dir) : []; + const receipts = files.length ? (JSON.parse(fs.readFileSync(path.join(dir, files[0]!), 'utf8')).receipts as FreeReceipt[]) : []; + return { line, files, receipts, keyed }; + } + + test('a clean shard writes one record per keyed passing file with the run identity', async () => { + const r = await record(passed()); + expect(r.files).toEqual(['free-receipts-shard-1.json']); + expect(r.receipts).toEqual([A, B].map(file => ({ file, key: KEYS[file]!, run_id: 100, run_attempt: 1, head_sha: HEAD, tree: TREE, shard: 1, written_at: NOW.toISOString() }))); + expect(r.line).toContain('2 written'); + }); + + test('a shard with a flaky retry, a crash or an unattributed failure writes nothing at all', async () => { + const retry = passed({ shard: 2, files: [B] }); + for (const [outcome, again] of [ + [passed({ status: 'failed', exitCode: 1, failingFiles: [B] }), retry], + [passed({ crashedFiles: [B] }), null], + [passed({ status: 'failed', exitCode: 1, unattributedFailures: 1 }), null], + [passed({ summary: { testsRan: 9, filesRan: 2, sawTerminalSummary: true } }), null], + ] as Array<[FreeShardOutcome, FreeShardOutcome | null]>) { + const r = await record(outcome, again); + expect(r.files).toEqual([]); + expect(r.keyed).toBe(0); + expect(r.line).toContain('none written'); + } + }); + + test('a pull_request run, a re-run attempt or a checkout that is not the run commit writes nothing', async () => { + for (const env of [{ ...ENV, GITHUB_EVENT_NAME: 'pull_request' }, { ...ENV, GITHUB_RUN_ATTEMPT: '2' }, { ...ENV, GITHUB_SHA: 'c'.repeat(40) }, {}]) { + const r = await record(passed(), null, env); + expect(r.files).toEqual([]); + expect(r.line).toContain('none written'); + } + }); +}); + +describe('shadowReport: one line per free file, nothing skipped', () => { + const accepted = (receipts: FreeReceipt[]) => verifyReceipts(receipts, check(world())); + + test('covers every free file exactly once; only a verified receipt with an equal key would be reused', () => { + const r = shadowReport({ + files: [A, B, C, D], + receipts: accepted([receipt(A), receipt(B, { key: '9'.repeat(64) }), receipt(C)]), + keys: keyReport([{ test: A }, { test: B }, { test: C, reasons: ['no trace'] }, { test: D }]), + durations: { [A]: 3000, [B]: 1000, [C]: 1000, [D]: 5000 }, + }); + expect(r.lines).toEqual([ + `${A} would-reuse 100 key=${KEYS[A]!.slice(0, 12)}`, + `${B} execute (key mismatch: receipt 999999999999, tree 222222222222)`, + `${C} execute (ineligible: no trace)`, + `${D} execute (no receipt)`, + ]); + expect(r.lines.map(l => l.split(' ')[0]).sort()).toEqual([A, B, C, D]); + expect(r.summary).toContain('would reuse 1 of 4 free files (25.0%)'); + expect(r.summary).toContain('3.0 s of 10.0 s seed time (30.0%)'); + }); + + test('a free file listed twice fails the exactly-once assertion', () => { + expect(() => shadowReport({ files: [A, A, B], receipts: accepted([receipt(A)]), keys: keyReport([{ test: A }, { test: B }]), durations: {} })).toThrow(/exactly once/); + }); + + test('refused receipts name their reason; unavailable receipts make every line execute', () => { + const refused = shadowReport({ files: [A], receipts: verifyReceipts([receipt(A)], check(world({ run: { event: 'pull_request' } }))), keys: keyReport([{ test: A }]), durations: {} }); + expect(refused.lines[0]).toStartWith(`${A} execute (not_push: `); + const down = shadowReport({ files: [A, B], receipts: { unavailable: 'HTTP 502' }, keys: keyReport([{ test: A }, { test: B }]), durations: {} }); + expect(down.lines).toEqual([`${A} execute (receipts unavailable: HTTP 502)`, `${B} execute (receipts unavailable: HTTP 502)`]); + expect(down.summary).toContain('would reuse 0 of 2'); + }); + + test('runShadowSelect with no receipts directory reports every file as execute', async () => { + const r = await runShadowSelect({ root: '/nowhere', files: [A, B], receiptsDir: path.join(tmp(), 'absent'), base: 'origin/main', durations: {}, now: NOW, keysAt: async (_t, _b, tests) => keyReport(tests.map(test => ({ test }))) }); + expect(r.lines).toHaveLength(2); + expect(r.lines.every(l => l.includes('execute (receipts unavailable: no receipts directory'))).toBe(true); + }); +}); + +describe('CLI flags', () => { + test('--receipts rides --ci-run or --shadow-select; --shadow-select needs --receipts and --base', () => { + expect(parseCliOptions(['--ci-run', 'p.json', '--shard', '1', '--result', 'r.json', '--receipts', 'out']).receipts).toBe('out'); + const shadow = parseCliOptions(['--shadow-select', '--receipts', 'in', '--base', 'origin/main']); + expect([shadow.shadowSelect, shadow.receipts, shadow.base]).toEqual([true, 'in', 'origin/main']); + expect(() => parseCliOptions(['--receipts', 'out'])).toThrow(); + expect(() => parseCliOptions(['--shadow-select', '--receipts', 'in'])).toThrow(); + expect(() => parseCliOptions(['--shadow-select', '--base', 'origin/main'])).toThrow(); + expect(() => parseCliOptions(['--shadow-select', '--receipts', 'in', '--base', 'm', '--ci-run', 'p.json', '--shard', '1', '--result', 'r.json'])).toThrow(); + expect(() => parseCliOptions(['--windows-only', '--ci-run', 'p.json', '--shard', '1', '--result', 'r.json', '--receipts', 'out'])).toThrow(); + }); +});