Merge remote-tracking branch 'origin/main' into capy/fw3-a3

scripts/exercised-map.ts

@@ -0,0 +1,384 @@
+#!/usr/bin/env bun
+/**
+ * test:exercised-map — maintain test/helpers/exercised-commands.json, the
+ * observed half of the exercised-path consumer map (scripts/pr-dependencies.ts
+ * exercisedConsumers; shadow mode, docs/TESTING_INTERNALS.md#exercised-path-map).
+ *
+ * Refreshes are union-only: an observation adds commands, never removes one.
+ * A refresh whose input is unavailable or partial writes nothing, keeps the
+ * committed union and exits non-zero with a stable code naming what was
+ * missing: 2 usage, 3 input unavailable (missing directory, failed or HTTP 403
+ * download), 4 input incomplete (no shards, no trial outcomes, an executed case
+ * without a shard artifact, a missing or malformed exercised-commands.json).
+ * Removing a command is a reviewed edit of the JSON file.
+ *
+ * Usage:
+ *   bun run test:exercised-map --from-run <id> [--out <file>]
+ *       download that census run's exercised-map artifact and union it into the committed map
+ *   bun run test:exercised-map --from-dir <downloaded exercised-map artifact dir> [--out <file>]
+ *       the same from a locally downloaded artifact (the fallback when the download hits HTTP 403)
+ *   bun run test:exercised-map --from-dir <dir> [--from-dir <dir>...] --run-id <id> [--out <file>] [--report <file>]
+ *       read paid-slice / gate-census report directories (shards/<shard>/...: transcripts, junit.xml,
+ *       PTY terminal.visible.log, trial-outcomes.jsonl) and union their commands into the map
+ *   bun run test:exercised-map --check
+ *       validate the committed map
+ *   bun run test:exercised-map --replay [<fixture>]
+ *       replay recorded red PR runs (default test/fixtures/exercised-map-replay.json) through
+ *       today's selector and its shadow; a MISS is a failed gate case the shadow would not run
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import {
+  checkoutExercisedConsumers, EXERCISED_MAP_PATH, parseExercisedMap, type ExercisedCase, type ExercisedConsumers, type ExercisedMap,
+} from './pr-dependencies';
+import { caseOwnerFile, EXERCISED_MAP_ANCHOR, PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from './test-pr-profile';
+import { CASE_TEST_NAMES } from './lib/paid-cases';
+import { computePaidCaseSelection } from './test-paid-shards';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+export const EXERCISED_MAP_ARTIFACT = 'exercised-map';
+
+/** Commands one shard directory recorded: per Bun test name, and shard-level (no test name). */
+export interface ShardCommands { shard: string; executed: string[]; perTest: Record<string, string[]>; shardLevel: string[] }
+
+function walk(dir: string): string[] {
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]);
+}
+
+function toolCommands(node: unknown, out: string[]): void {
+  if (Array.isArray(node)) { for (const item of node) toolCommands(item, out); return; }
+  if (!node || typeof node !== 'object') return;
+  const record = node as Record<string, any>;
+  if (record.type === 'tool_use' && typeof record.input?.command === 'string') out.push(record.input.command);
+  for (const value of Object.values(record)) if (value && typeof value === 'object') toolCommands(value, out);
+}
+
+/** Every `shards/<shard>/` directory under `dir`, with the Bash commands its transcripts, JSONL logs and PTY screens recorded. */
+export function readShardCommands(dir: string): ShardCommands[] {
+  if (!fs.existsSync(dir)) return [];
+  const shardDirs = walk(dir).map(file => path.relative(dir, file).split(path.sep)).flatMap(parts => {
+    const at = parts.lastIndexOf('shards');
+    return at >= 0 && parts.length > at + 2 ? [path.join(dir, ...parts.slice(0, at + 2))] : [];
+  });
+  return [...new Set(shardDirs)].sort().map(shardDir => {
+    const record: ShardCommands = { shard: path.basename(shardDir), executed: [], perTest: {}, shardLevel: [] };
+    for (const file of walk(shardDir)) {
+      const base = path.basename(file);
+      try {
+        if (base === 'junit.xml') {
+          for (const match of fs.readFileSync(file, 'utf8').matchAll(/<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) {
+            const name = /\bname="([^"]*)"/.exec(match[1]!)?.[1];
+            if (name && !/<skipped/.test(match[3] ?? '')) record.executed.push(name.replace(/&quot;/g, '"').replace(/&amp;/g, '&'));
+          }
+        } else if (base.endsWith('.json')) {
+          const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+          if (Array.isArray(parsed?.tests) && parsed.tests.some((test: any) => test?.transcript)) {
+            for (const test of parsed.tests) { const out: string[] = []; toolCommands(test.transcript ?? [], out); if (out.length) (record.perTest[test.name] ??= []).push(...out); }
+          } else toolCommands(parsed, record.shardLevel);
+        } else if (base.endsWith('.jsonl')) {
+          for (const line of fs.readFileSync(file, 'utf8').split('\n')) if (line.includes('tool_use')) { try { toolCommands(JSON.parse(line), record.shardLevel); } catch { /* partial line */ } }
+        } else if (base === 'terminal.visible.log') {
+          for (const match of fs.readFileSync(file, 'utf8').matchAll(/Bash\(([^\r\n]{0,400})/g)) record.shardLevel.push(`PTY:${match[1]}`);
+        }
+      } catch { /* unreadable evidence file: its commands are simply not observed */ }
+    }
+    record.shardLevel = [...new Set(record.shardLevel)];
+    for (const name of Object.keys(record.perTest)) record.perTest[name] = [...new Set(record.perTest[name])];
+    return record;
+  });
+}
+
+const testNameCase = (name: string, ids: string[]): string | null =>
+  ids.includes(name) ? name : Object.keys(CASE_TEST_NAMES).find(id => CASE_TEST_NAMES[id] === name && ids.includes(id)) ?? ids.find(id => name.includes(id)) ?? null;
+
+/**
+ * Raw commands per case. A case shard (`<stem>--<id>`) is that case; a file
+ * shard's per-test commands go to the case its test name names, and its
+ * file-level commands to every case of the file that ran (by JUnit name, else
+ * all of them): conservative over-attribution, as in the measured prototype.
+ */
+export function attributeCommands(shards: ShardCommands[], maps = PR_PROFILE_MAPS): Map<string, string[]> {
+  const ids = Object.keys(maps.e2eTouchfiles);
+  const byFile = new Map<string, string[]>();
+  for (const id of ids) { const owner = caseOwnerFile(id, maps); if (owner) byFile.set(owner, [...byFile.get(owner) ?? [], id]); }
+  const out = new Map<string, string[]>();
+  const add = (id: string, commands: string[]) => out.set(id, [...out.get(id) ?? [], ...commands]);
+  for (const shard of shards) {
+    const [stem, caseKey] = shard.shard.split('--');
+    const caseId = caseKey?.replace(/\.t\d+$/, '');
+    const owned = byFile.get(`test/${stem}.test.ts`) ?? [];
+    const ran = owned.filter(id => shard.executed.some(name => testNameCase(name, [id]) === id));
+    const targets = caseId && ids.includes(caseId) ? [caseId] : ran.length ? ran : owned;
+    for (const id of targets) add(id, shard.shardLevel);
+    for (const [name, commands] of Object.entries(shard.perTest)) {
+      const id = testNameCase(name, targets) ?? testNameCase(name, ids);
+      for (const target of id ? [id] : targets) add(target, commands);
+    }
+  }
+  return out;
+}
+
+/** Known trial cost per case summed over the trial-outcomes.jsonl files under `dir` (executed trials only). */
+export function readCaseCosts(dir: string): Map<string, number> {
+  const out = new Map<string, number>();
+  if (!fs.existsSync(dir)) return out;
+  for (const file of walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')) {
+    for (const line of fs.readFileSync(file, 'utf8').split('\n').filter(Boolean)) {
+      try {
+        const record = JSON.parse(line);
+        if (record.execution === 'executed' && record.cost_known !== false && typeof record.case === 'string') out.set(record.case, (out.get(record.case) ?? 0) + (Number(record.cost_usd) || 0));
+      } catch { /* malformed line */ }
+    }
+  }
+  return out;
+}
+
+/** One run's observations as a map: the bins each attributed case invoked, and its known cost. */
+export function observeRun(commandsByCase: Map<string, string[]>, ex: Pick<ExercisedConsumers, 'binsInCommand'>, run: { runId: string; observedAt: string; costs?: Map<string, number> }): ExercisedMap {
+  const cases: Record<string, ExercisedCase> = {};
+  for (const [id, commands] of commandsByCase) {
+    cases[id] = { observations: 1, withCommands: commands.length ? 1 : 0, commands: [...new Set(commands.flatMap(command => ex.binsInCommand(command)))].sort() };
+  }
+  for (const [id, cost] of run.costs ?? []) {
+    cases[id] ??= { observations: 1, withCommands: 0, commands: [] };
+    Object.assign(cases[id], { costUsd: Math.round(cost * 10000) / 10000, costRuns: 1 });
+  }
+  return { schema: 1, generatedAt: run.observedAt, observedThrough: run.observedAt, sourceRuns: [run.runId], cases };
+}
+
+/**
+ * Union-only merge: commands are unioned, counts add for runs the base has
+ * not seen, costs average by run count, and nothing the base holds is ever
+ * removed (an absent or empty case in `fresh` changes nothing).
+ */
+export function mergeExercisedMap(base: ExercisedMap, fresh: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap {
+  const counts = fresh.sourceRuns.some(run => !base.sourceRuns.includes(run));
+  const cases: Record<string, ExercisedCase> = structuredClone(base.cases);
+  for (const [id, entry] of Object.entries(fresh.cases)) {
+    const old = cases[id];
+    const merged: ExercisedCase = {
+      observations: (old?.observations ?? 0) + (counts ? entry.observations : 0),
+      withCommands: (old?.withCommands ?? 0) + (counts ? entry.withCommands : 0),
+      commands: [...new Set([...old?.commands ?? [], ...entry.commands])].sort(),
+    };
+    const oldRuns = old?.costRuns ?? 0, freshRuns = counts && entry.costUsd !== undefined ? entry.costRuns ?? 1 : 0;
+    if (freshRuns) Object.assign(merged, { costUsd: Math.round(((old?.costUsd ?? 0) * oldRuns + entry.costUsd! * freshRuns) / (oldRuns + freshRuns) * 10000) / 10000, costRuns: oldRuns + freshRuns });
+    else if (old?.costUsd !== undefined) Object.assign(merged, { costUsd: old.costUsd, costRuns: oldRuns });
+    cases[id] = merged;
+  }
+  return {
+    schema: 1, generatedAt,
+    observedThrough: [base.observedThrough, fresh.observedThrough].sort().at(-1)!,
+    sourceRuns: [...new Set([...base.sourceRuns, ...fresh.sourceRuns])].sort(),
+    cases: Object.fromEntries(Object.entries(cases).sort(([a], [b]) => a.localeCompare(b))),
+  };
+}
+
+export function formatExercisedMap(map: ExercisedMap): string {
+  return `${JSON.stringify(map, null, 2)}\n`;
+}
+
+/** `MAP_GAP <case> <command>`: a recorded command whose files all lie outside the case's static-plus-observed exercised closure. */
+export function mapGaps(commandsByCase: Map<string, string[]>, ex: ExercisedConsumers): Array<{ id: string; command: string }> {
+  const gaps: Array<{ id: string; command: string }> = [];
+  for (const [id, commands] of [...commandsByCase].sort(([a], [b]) => a.localeCompare(b))) {
+    const closure = ex.closure(id);
+    for (const command of [...new Set(commands.flatMap(raw => ex.binsInCommand(raw)))].sort()) {
+      const files = ex.commandFiles(command);
+      if (files.length && !files.some(file => closure.has(file))) gaps.push({ id, command });
+    }
+  }
+  return gaps;
+}
+
+/** Executed cases that recorded no command although their loaded skill's shell runs bins: blind to gap detection. */
+export function blindCases(commandsByCase: Map<string, string[]>, ex: ExercisedConsumers): string[] {
+  return [...commandsByCase].filter(([id, commands]) => commands.length === 0
+    && [...ex.closure(id)].some(([file, label]) => file.startsWith('bin/') && label.startsWith('skill bash via'))).map(([id]) => id).sort();
+}
+
+/** Stable exit codes of test:exercised-map (docs/TESTING_INTERNALS.md#exercised-path-map). */
+export const REFRESH_EXIT = { usage: 2, unavailable: 3, incomplete: 4 } as const;
+
+/** A refresh that cannot be trusted: nothing is written and the committed union stays as it was. */
+export class RefreshError extends Error {
+  constructor(readonly exitCode: typeof REFRESH_EXIT.unavailable | typeof REFRESH_EXIT.incomplete, message: string) { super(message); }
+}
+
+/** One MAP_GAP line, saying who it is for and what follows from it. */
+export function formatMapGap(gap: { id: string; command: string }): string {
+  return `MAP_GAP ${gap.id} ${gap.command}: non-blocking for this PR; the next census refresh widens the map (${EXERCISED_MAP_ANCHOR})`;
+}
+
+function writeAtomically(file: string, text: string): void {
+  fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
+  const temp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(temp, text);
+  fs.renameSync(temp, file);
+}
+
+/**
+ * Census refresh from local report directories (paid-slice artifacts under
+ * shards/, plus trial-outcomes.jsonl). Every executed case must have shard
+ * evidence; a missing directory, no shards, no trial outcomes or an executed
+ * case without a shard throws before anything is written.
+ */
+export function refreshFromReportDirs(dirs: string[], runId: string, base: ExercisedMap, ex: ExercisedConsumers, observedAt = new Date().toISOString(), maps = PR_PROFILE_MAPS) {
+  const missing = dirs.filter(dir => !fs.existsSync(dir));
+  if (missing.length) throw new RefreshError(REFRESH_EXIT.unavailable, `report director${missing.length > 1 ? 'ies' : 'y'} not found: ${missing.join(', ')}`);
+  const shards = dirs.flatMap(readShardCommands);
+  if (shards.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no shards/<shard>/ directories under ${dirs.join(', ')}; download every paid-slice-* artifact of the run first`);
+  const outcomes = dirs.flatMap(dir => walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl'));
+  if (outcomes.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no trial-outcomes.jsonl under ${dirs.join(', ')}, so the executed cases cannot be checked against the shard evidence`);
+  const commands = attributeCommands(shards, maps);
+  const executed = new Set(outcomes.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => {
+    try { const record = JSON.parse(line); return record.execution === 'executed' && typeof record.case === 'string' ? [record.case as string] : []; } catch { return []; }
+  })));
+  const uncovered = [...executed].filter(id => Object.hasOwn(maps.e2eTouchfiles, id) && !commands.has(id)).sort();
+  if (uncovered.length) throw new RefreshError(REFRESH_EXIT.incomplete, `${uncovered.length} executed case(s) have no shard artifact: ${uncovered.slice(0, 10).join(', ')}${uncovered.length > 10 ? ' ...' : ''}`);
+  const costs = new Map<string, number>();
+  for (const dir of dirs) for (const [id, cost] of readCaseCosts(dir)) costs.set(id, (costs.get(id) ?? 0) + cost);
+  const merged = mergeExercisedMap(base, observeRun(commands, ex, { runId, observedAt, costs }), observedAt);
+  return { merged, commands };
+}
+
+function census(dirs: string[], runId: string, out: string, reportPath: string | undefined): void {
+  const ex = checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES);
+  const base = ex.map!;
+  const { merged, commands } = refreshFromReportDirs(dirs, runId, base, ex);
+  writeAtomically(out, formatExercisedMap(merged));
+  const added = Object.entries(merged.cases).flatMap(([id, entry]) => entry.commands.filter(command => !base.cases[id]?.commands.includes(command)).map(command => `${id} ${command}`));
+  const gaps = mapGaps(commands, ex);
+  const blind = blindCases(commands, ex);
+  const lines = [
+    '## Exercised-path map (shadow, non-blocking)', '',
+    `- ${commands.size} executed case(s), ${[...commands.values()].reduce((sum, list) => sum + list.length, 0)} recorded command(s); ${added.length} new observed edge(s).`,
+    `- Refresh: \`bun run test:exercised-map --from-run ${runId}\`, then open the change as an ordinary PR (shadow-only data: it selects no paid case).`,
+    `- MAP_GAP (a command outside its case's static-plus-observed closure): ${gaps.length}`,
+    ...gaps.map(gap => `  - ${formatMapGap(gap)}`),
+    `- Cases that recorded no commands although their skill runs bins (PTY capture is the activation prerequisite): ${blind.length ? blind.join(', ') : 'none'}`,
+    ...added.slice(0, 50).map(edge => `  - new: ${edge}`),
+    `- Docs: ${EXERCISED_MAP_ANCHOR}`,
+  ];
+  console.log(lines.join('\n'));
+  if (reportPath) fs.writeFileSync(reportPath, `${lines.join('\n')}\n`);
+}
+
+/** Union every exercised-commands.json of a downloaded exercised-map artifact into `base`; any missing or malformed file throws. */
+export function mergeArtifactDir(dir: string, base: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap {
+  const files = fs.existsSync(dir) ? walk(dir).filter(file => path.basename(file) === 'exercised-commands.json') : [];
+  if (files.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no exercised-commands.json in ${dir} (the census refresh step failed or its ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact is partial)`);
+  let map = base;
+  for (const file of files.sort()) {
+    let fresh: ExercisedMap;
+    try { fresh = parseExercisedMap(fs.readFileSync(file, 'utf8')); }
+    catch (error) { throw new RefreshError(REFRESH_EXIT.incomplete, `${path.relative(dir, file)} is not a complete exercised map: ${error instanceof Error ? error.message : String(error)}`); }
+    map = mergeExercisedMap(map, fresh, generatedAt);
+  }
+  return map;
+}
+
+export type ArtifactDownload = (runId: string, dir: string) => { status: number | null; stderr: string };
+
+const ghDownload: ArtifactDownload = (runId, dir) => {
+  const result = spawnSync('gh', ['run', 'download', runId, '--pattern', `${EXERCISED_MAP_ARTIFACT}-*`, '--dir', dir], { encoding: 'utf8', timeout: 120_000 });
+  return { status: result.status, stderr: result.stderr || result.error?.message || '' };
+};
+
+/** Download a census run's exercised-map artifact and union it into `target`; on any failure nothing is written. */
+export function refreshFromRun(runId: string, target: string, download: ArtifactDownload = ghDownload): ExercisedMap {
+  if (!/^\d+$/.test(runId)) throw new RefreshError(REFRESH_EXIT.unavailable, `--from-run needs a numeric GitHub run id, got ${runId}`);
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-'));
+  try {
+    const result = download(runId, dir);
+    if (result.status !== 0) {
+      const reason = result.stderr.trim();
+      const hint = /\b403\b/.test(reason)
+        ? `HTTP 403 (the installation's artifact download limit); download the ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact from the run page and pass its directory with --from-dir`
+        : `check \`gh auth status\` and that run ${runId} uploaded an ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact`;
+      throw new RefreshError(REFRESH_EXIT.unavailable, `gh run download ${runId} failed: ${reason || `exit ${result.status}`}; ${hint}`);
+    }
+    const map = mergeArtifactDir(dir, parseExercisedMap(fs.readFileSync(target, 'utf8')));
+    writeAtomically(target, formatExercisedMap(map));
+    return map;
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+}
+
+export const REPLAY_FIXTURE = 'test/fixtures/exercised-map-replay.json';
+
+export interface ReplayRun { runId: number; pr: number; packageVersionOnly: boolean; failedGateCases: string[]; changedFiles: string[] }
+
+/** One recorded red run under today's selection and its shadow: which failed gate cases each would run. */
+export interface ShadowReplayRow { runId: number; pr: number; mode: string; today: number; shadow: number; savingUsd: number; caughtToday: string[]; misses: string[]; retired: string[] }
+
+export function replayShadow(runs: readonly ReplayRun[]): ShadowReplayRow[] {
+  const gateIds = Object.keys(PR_PROFILE_MAPS.tiers).filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate');
+  return runs.map(run => {
+    const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: run.changedFiles, packageVersionOnly: run.packageVersionOnly });
+    const today = new Set(result.selection.e2e ?? gateIds);
+    const shadow = result.coverage?.shadow;
+    if (shadow?.error) throw new Error(`run ${run.runId}: shadow unavailable (${shadow.error})`);
+    const shadowSet = new Set(shadow?.e2e ?? today);
+    const live = run.failedGateCases.filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate');
+    return { runId: run.runId, pr: run.pr, mode: result.coverage?.mode ?? 'full', today: today.size, shadow: shadowSet.size, savingUsd: shadow?.predictedSavingUsd ?? 0,
+      caughtToday: live.filter(id => today.has(id)), misses: live.filter(id => !shadowSet.has(id)), retired: run.failedGateCases.filter(id => !live.includes(id)) };
+  });
+}
+
+export function formatShadowReplay(rows: readonly ShadowReplayRow[]): string[] {
+  const pairs = (list: readonly ShadowReplayRow[]) => list.reduce((sum, row) => sum + new Set([...row.caughtToday, ...row.misses]).size, 0);
+  const misses = rows.flatMap(row => row.misses.map(id => `${row.runId} #${row.pr} ${id}`));
+  const modes = new Map<string, number>();
+  for (const row of rows) modes.set(row.mode, (modes.get(row.mode) ?? 0) + 1);
+  const mean = (pick: (row: ShadowReplayRow) => number) => rows.length ? (rows.reduce((sum, row) => sum + pick(row), 0) / rows.length).toFixed(1) : 'n/a';
+  return [
+    `Exercised-path shadow replay over ${rows.length} recorded red PR run(s) on this checkout`,
+    `- failed gate (run, case) pairs: ${pairs(rows)}, ${pairs(rows.filter(row => row.mode !== 'full-fallback'))} of them in runs that do not fall back to the full gate (${rows.reduce((sum, row) => sum + row.retired.length, 0)} more on cases no longer in the gate)`,
+    `- today's selection runs ${rows.reduce((sum, row) => sum + row.caughtToday.length, 0)} of them; modes ${[...modes].map(([mode, n]) => `${mode} ${n}`).join(', ')}`,
+    `- mean gate cases: today ${mean(row => row.today)}, shadow ${mean(row => row.shadow)}; predicted shadow saving over these runs $${rows.reduce((sum, row) => sum + row.savingUsd, 0).toFixed(2)}`,
+    `- MISSES (failed, not in the shadow selection): ${misses.length}`,
+    ...misses.map(miss => `  - ${miss}`),
+  ];
+}
+
+const USAGE = 'usage: bun run test:exercised-map --from-run <id> [--out <file>] | --from-dir <dir>... [--run-id <id>] [--out <file>] [--report <file>] | --check | --replay [<fixture>]';
+
+if (import.meta.main) {
+  const argv = process.argv.slice(2);
+  const values = (flag: string) => argv.flatMap((arg, index) => arg === flag && argv[index + 1] ? [argv[index + 1]!] : []);
+  const out = values('--out')[0] ?? path.join(ROOT, EXERCISED_MAP_PATH);
+  try {
+    if (argv.includes('--check')) {
+      const map = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8'));
+      console.log(`${EXERCISED_MAP_PATH}: ${Object.keys(map.cases).length} cases, ${map.sourceRuns.length} source runs, observed through ${map.observedThrough}`);
+    } else if (argv.includes('--replay')) {
+      const fixture = JSON.parse(fs.readFileSync(path.resolve(ROOT, values('--replay')[0] ?? REPLAY_FIXTURE), 'utf8')) as { runs: ReplayRun[] };
+      const rows = replayShadow(fixture.runs);
+      console.log(formatShadowReplay(rows).join('\n'));
+      if (rows.some(row => row.misses.length)) process.exit(1);
+    } else if (values('--from-run').length) {
+      const runId = values('--from-run')[0]!;
+      const map = refreshFromRun(runId, out);
+      console.log(`union-merged run ${runId}'s ${EXERCISED_MAP_ARTIFACT} artifact into ${path.relative(ROOT, out)} (${Object.keys(map.cases).length} cases, observed through ${map.observedThrough}); review the diff and open it as a PR`);
+    } else if (values('--from-dir').length) {
+      const dirs = values('--from-dir');
+      if (dirs.length === 1 && fs.existsSync(dirs[0]!) && readShardCommands(dirs[0]!).length === 0 && walk(dirs[0]!).some(file => path.basename(file) === 'exercised-commands.json')) {
+        const map = mergeArtifactDir(dirs[0]!, parseExercisedMap(fs.readFileSync(out, 'utf8')));
+        writeAtomically(out, formatExercisedMap(map));
+        console.log(`union-merged the ${EXERCISED_MAP_ARTIFACT} artifact in ${dirs[0]} into ${path.relative(ROOT, out)}; review the diff and open it as a PR`);
+      } else {
+        const runId = values('--run-id')[0];
+        if (!runId) { console.error(`test:exercised-map: --from-dir with report directories needs --run-id <id> (the run whose artifacts these are)\n${USAGE}`); process.exit(REFRESH_EXIT.usage); }
+        census(dirs, runId, out, values('--report')[0]);
+      }
+    } else {
+      console.error(USAGE);
+      process.exit(REFRESH_EXIT.usage);
+    }
+  } catch (error) {
+    console.error(`test:exercised-map: ${error instanceof Error ? error.message : String(error)}${error instanceof RefreshError ? '; nothing was written, the committed union is unchanged' : ''}`);
+    process.exit(error instanceof RefreshError ? error.exitCode : 1);
+  }
+}

scripts/harness-version.json

@@ -12,11 +12,11 @@
   "files": {
     "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5},
     "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5},
-    "scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":5},
+    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
     "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
     "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
     "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
-    "scripts/test-pr-profile.ts": {"blob":"ddc1e5991636c8c94722cd32472edce26489b26d","since":5},
+    "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
     "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
     "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
     "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},

scripts/lib/paid-report.ts

@@ -22,6 +22,9 @@ import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, typ
 import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
 import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
 import type { RunShardsOptions } from '../test-paid-shards';
+import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map';
+import { checkoutExercisedConsumers } from '../pr-dependencies';
+import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile';
 
 // ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
 
@@ -482,6 +485,30 @@ export function infraOnly(problems: readonly string[]): boolean {
   return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem)));
 }
 
+/**
+ * Shadow gap detection (docs/TESTING_INTERNALS.md#exercised-path-map): a
+ * command an executed case ran outside its static-plus-observed exercised
+ * closure prints a MAP_GAP line (formatMapGap: non-blocking for this PR, the
+ * next census refresh widens the map) and lands in map-gaps.txt. It is its own
+ * result: never a case verdict, a lane problem or a re-dispatch input.
+ */
+export function reportMapGaps(reportDir: string, executedCases: ReadonlySet<string>, rootDir = ROOT): string[] {
+  const out = path.join(reportDir, 'map-gaps.txt');
+  try {
+    const commands = new Map([...attributeCommands(readShardCommands(reportDir))].filter(([id, list]) => executedCases.has(id) && list.length > 0));
+    const lines = commands.size === 0 ? [] : mapGaps(commands, checkoutExercisedConsumers(PR_PROFILE_MAPS, rootDir, PR_FULL_GATE_FILES)).map(formatMapGap);
+    fs.writeFileSync(out, lines.length ? `${lines.join('\n')}\n` : '');
+    console.log(`[test:paid] exercised-path map (shadow, non-blocking): ${lines.length} MAP_GAP line(s) over ${commands.size} executed case(s) with recorded commands`);
+    for (const line of lines) console.log(`  ${line}`);
+    return lines;
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    fs.writeFileSync(out, `MAP_GAP unavailable: ${reason}\n`);
+    console.log(`[test:paid] exercised-path map (shadow, non-blocking): MAP_GAP unavailable: ${reason}`);
+    return [];
+  }
+}
+
 /**
  * Report mode: reconcile slice artifacts against the manifest (fail-closed),
  * compute every panel verdict with panelVerdict(), write collector-outcomes
@@ -743,6 +770,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
       ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
   // evals.yml wraps these in a fixed fence for the PR comment.
   failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
+  reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir);
   if (verdict.problems.length) {
     console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
     for (const problem of verdict.problems) console.error(`  ✗ ${problem}`);

scripts/pr-dependencies.ts

@@ -49,6 +49,12 @@ export interface DerivedDependencies {
   tracked: Set<string>;
   /** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */
   trace(caseId: string, file: string): string[] | null;
+  /**
+   * Files whose import scan failed for a reason other than a leading `#!`
+   * line (their imports are unknown). PR selection restores the full gate
+   * while this is non-empty; it fills as the derivation reads files.
+   */
+  scanErrors: Map<string, string>;
   /**
    * Tracked files whose reference text still names `missing`, a path absent
    * from the tree (deleted in the diff): its path or extensionless import
@@ -60,8 +66,10 @@ export interface DerivedDependencies {
 }
 
 const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/;
+/** An extensionless script Bun or Node runs: its imports are code, not shell. */
+const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/;
 const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/;
-const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
+const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
 /** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */
 const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g;
 const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g;
@@ -82,10 +90,31 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
   'scripts/build.sh',
   // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
   'lib/result-codes.ts', 'lib/result-codes-release.ts',
+  // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
+  'test/helpers/exercised-commands.json',
 ];
 const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts';
 
-const transpiler = new Bun.Transpiler({ loader: 'tsx' });
+const transpilers = {
+  ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }),
+  js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }),
+};
+
+/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */
+export function isCodeFile(file: string, text: string): boolean {
+  return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text));
+}
+
+/**
+ * Relative import specifiers of a code file. A leading `#!` line is blanked
+ * first (`scanImports` rejects it), and each extension gets its own loader
+ * (the TSX loader misreads `.ts` generics and casts). Any other parse error
+ * throws, so callers fail closed.
+ */
+export function scanCodeImports(file: string, text: string): string[] {
+  const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts';
+  return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path);
+}
 
 /**
  * Text a reference scan reads: whole-line comments are prose about other
@@ -168,15 +197,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
   };
   const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));
 
+  const scanErrors = new Map<string, string>();
+  const importsOf = (file: string): string[] => {
+    try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; }
+  };
   const edges = new Map<string, string[]>();
   const refs = (file: string): string[] => {
     if (edges.has(file)) return edges.get(file)!;
     const raw = read(file);
     const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw);
     const out = new Set<string>();
-    if (CODE.test(file)) {
-      try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ }
-    }
+    if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); }
     for (const match of text.matchAll(REPO_PATH)) {
       const ref = match[1]!.replace(/[.]+$/, '');
       if (trackedSet.has(ref)) out.add(ref);
@@ -238,19 +269,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
     const file = pending.pop()!;
     if (global.has(file)) continue;
     global.add(file);
-    if (!CODE.test(file)) continue;
-    try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ }
+    if (!isCodeFile(file, read(file))) continue;
+    for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); }
   }
   const importTargets = new Map<string, string[]>();
   const targetsOf = (file: string): string[] => {
     if (!importTargets.has(file)) {
       const targets: string[] = [];
-      if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
-        try {
-          for (const item of transpiler.scanImports(read(file))) {
-            if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path)));
-          }
-        } catch { /* literal mentions below still apply */ }
+      if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
+        for (const specifier of importsOf(file)) {
+          if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier)));
+        }
       }
       importTargets.set(file, targets);
     }
@@ -287,7 +316,7 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
     while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!);
     return chain;
   };
-  return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers };
+  return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors };
 }
 
 const cache = new Map<string, DerivedDependencies>();
@@ -299,3 +328,295 @@ export function derivedDependencies(maps: DependencyMaps, root: string): Derived
   return cache.get(key)!;
 }
 
+// ─── Exercised-path consumer map (W2-A2, shadow mode) ──────────────────────
+
+/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */
+export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json';
+
+export interface ExercisedCase {
+  /** Recorded trials (run, case) the commands were taken from. */
+  observations: number;
+  /** Of those, trials that recorded at least one command. */
+  withCommands: number;
+  /** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */
+  commands: string[];
+  /** Mean known trial cost per run, for the shadow cost estimate. */
+  costUsd?: number;
+  costRuns?: number;
+}
+
+export interface ExercisedMap {
+  schema: 1;
+  generatedAt: string;
+  observedThrough: string;
+  sourceRuns: string[];
+  cases: Record<string, ExercisedCase>;
+}
+
+export function isExercisedMap(value: unknown): value is ExercisedMap {
+  const map = value as ExercisedMap;
+  return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string'
+    && Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string')
+    && !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry
+      && Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands)
+      && Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string'));
+}
+
+/** Parse an exercised map's text; throws on malformed input so callers fail closed. */
+export function parseExercisedMap(text: string): ExercisedMap {
+  const parsed = JSON.parse(text);
+  if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`);
+  return parsed;
+}
+
+export interface ExercisedConsumers {
+  /** Repository file -> case ids whose exercised closure contains it. */
+  e2e: Map<string, Set<string>>;
+  /** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via <file>, observed <command>, declared via <file>, host). */
+  closure(caseId: string): Map<string, string>;
+  /** Bins (repo paths) and compiled outfiles one recorded command invokes. */
+  binsInCommand(command: string): string[];
+  /** The tracked files a bin or compiled outfile runs. */
+  commandFiles(command: string): string[];
+  /** The observed map the consumers were built with. */
+  map: ExercisedMap | null;
+}
+
+const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/;
+const MESSAGE_CALL = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/;
+const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/;
+const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/;
+const INTERPRETER = /\b(?:bash|sh|zsh|bun|node|eval|source|python3?)\b/;
+
+/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */
+function substitutions(span: string): string[] {
+  const out: string[] = [];
+  for (let i = 0; i < span.length; i++) {
+    if (span[i] === '\\') { i++; continue; }
+    if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; }
+    if (span[i] !== '$' || span[i + 1] !== '(') continue;
+    let depth = 0, j = i + 1;
+    for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; }
+    out.push(span.slice(i, j + 1));
+    i = j;
+  }
+  return out;
+}
+
+/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */
+function stripMessages(text: string): string {
+  let out = '', i = 0;
+  while (i < text.length) {
+    const prev = i === 0 ? '\n' : text[i - 1]!;
+    const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null;
+    if (!call) { out += text[i]; i++; continue; }
+    let j = i + call[0].length, quote: string | null = null, depth = 0;
+    while (j < text.length) {
+      const c = text[j]!;
+      if (c === '\\') { j += 2; continue; }
+      if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; }
+      if (c === '"' || c === "'") { quote = c; j++; continue; }
+      if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; }
+      if (c === ')' && depth) { depth--; j++; continue; }
+      if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break;
+      j++;
+    }
+    out += ` ${substitutions(text.slice(i, j)).join('; ')} `;
+    i = j;
+  }
+  return out;
+}
+
+/**
+ * Executable shell text: comments, case patterns and message calls are
+ * prose. A heredoc an interpreter reads stays whole; an expanding (unquoted)
+ * heredoc keeps its command substitutions; a quoted one fed to anything else
+ * is dropped.
+ */
+export function executableShell(text: string): string {
+  return stripMessages(text
+    .replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) =>
+      INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n'))
+    .split('\n').filter(line => !/^\s*#(?!!)/.test(line))
+    .map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, ''))
+    .join('\n'));
+}
+
+function commandPosition(before: string): boolean {
+  const head = before.replace(CMD_PREFIX, '');
+  return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head));
+}
+
+/**
+ * Consumers by exercised paths: a case consumes its touchfile matches, the
+ * closure of its test file, the executable shell of its loaded skill
+ * (template and generated SKILL.md) and of every bin, lib and script that
+ * shell runs, and the bins its recorded trials invoked (`observed`). The
+ * closure stops at `stopAt` (the full-gate files) and the global set,
+ * which stay full.
+ */
+export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers {
+  const base = derivedDependencies(maps, root);
+  const tracked = [...base.tracked];
+  const texts = new Map<string, string>();
+  const read = (file: string): string => {
+    if (!texts.has(file)) {
+      let text = '';
+      try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ }
+      texts.set(file, text);
+    }
+    return texts.get(file)!;
+  };
+  const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file)));
+  const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));
+  const expandRef = (ref: string): string[] => {
+    const clean = ref.replace(/[.]+$/, '');
+    if (base.tracked.has(clean)) return [clean];
+    return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : [];
+  };
+  const expanded = new Map<string, string[]>();
+  const expand = (pattern: string): string[] => {
+    if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : [];
+    if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern)));
+    return expanded.get(pattern)!;
+  };
+  const binByName = new Map<string, string>();
+  for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) {
+    binByName.set(path.posix.basename(file), file);
+    binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file);
+  }
+  const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)]
+    .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) }));
+  const outfileHit = (outfile: string, text: string) => text.includes(outfile)
+    || (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text));
+  let scripts: Record<string, string> = {};
+  try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ }
+
+  const shellEdges = (text: string, from: string, out: Set<string>) => {
+    for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) {
+      const start = text.lastIndexOf('\n', match.index! - 1) + 1;
+      const bin = binByName.get(match[0]);
+      if (bin && commandPosition(text.slice(start, match.index))) out.add(bin);
+    }
+    for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) {
+      const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index);
+      if (/[\w.@-]$/.test(before)) continue;
+      if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file);
+    }
+    for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const file of expandRef(match[0])) out.add(file);
+    for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) {
+      for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file);
+      const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1);
+      if (!name) continue;
+      const sibling = path.posix.join(path.posix.dirname(from), name);
+      if (base.tracked.has(sibling)) out.add(sibling);
+      else if (binByName.has(name)) out.add(binByName.get(name)!);
+    }
+    if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup');
+    for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file);
+    for (const match of text.matchAll(PACKAGE_RUN)) {
+      const command = scripts[match[1]!];
+      if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file);
+    }
+  };
+  const edges = new Map<string, string[]>();
+  const execRefs = (file: string): string[] => {
+    if (edges.has(file)) return edges.get(file)!;
+    const raw = read(file);
+    const out = new Set<string>();
+    if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
+      if (isCodeFile(file, raw)) {
+        try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } }
+        catch (error) { base.scanErrors.set(file, String(error)); }
+        const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, '');
+        for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit);
+        for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit);
+        if (/join\([^)\n]*['"]setup['"]\s*\)/.test(code) && base.tracked.has('setup')) out.add('setup');
+        for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit);
+      } else if (isShell(file)) {
+        shellEdges(executableShell(raw), file, out);
+      } else if (/\.(?:md|tmpl)$/.test(file)) {
+        shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out);
+      }
+    }
+    out.delete(file);
+    edges.set(file, [...out]);
+    return edges.get(file)!;
+  };
+  const stopped = new Map<string, boolean>();
+  const stops = (file: string): boolean => {
+    if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern)));
+    return stopped.get(file)!;
+  };
+  const binsInCommand = (command: string): string[] => {
+    const out = new Set<string>();
+    const pty = command.startsWith('PTY:');
+    for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) {
+      let token = match[0];
+      if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; }
+      const at = match.index!;
+      if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue;
+      const bin = binByName.get(token);
+      if (bin) out.add(bin);
+    }
+    for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); }
+    for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile);
+    return [...out];
+  };
+  const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? [];
+
+  const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts'));
+  const closures = new Map<string, Map<string, string>>();
+  const caseClosure = (id: string): Map<string, string> => {
+    if (closures.has(id)) return closures.get(id)!;
+    const patterns = maps.e2eTouchfiles[id] ?? [];
+    const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern));
+    const declared = patterns.flatMap(expand);
+    const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file));
+    const touchfiles = new Set([...declared, ...generated]);
+    const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file);
+    const seeds: Array<[string, string]> = [
+      ...owners.map(file => [file, 'import'] as [string, string]),
+      ...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]),
+      ...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])),
+    ];
+    const ownerText = owners.map(read).join('\n');
+    for (const host of hostIds) {
+      if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']);
+    }
+    const reached = new Map<string, string>();
+    const queue = seeds.filter(([file]) => base.tracked.has(file));
+    for (let head = 0; head < queue.length; head++) {
+      const [file, label] = queue[head]!;
+      if (reached.has(file)) continue;
+      reached.set(file, touchfiles.has(file) ? 'touchfile' : label);
+      if (stops(file)) continue;
+      for (const next of execRefs(file)) {
+        if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue;
+        queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]);
+      }
+    }
+    closures.set(id, reached);
+    return reached;
+  };
+  const e2e = new Map<string, Set<string>>();
+  for (const id of Object.keys(maps.e2eTouchfiles)) {
+    for (const file of caseClosure(id).keys()) {
+      if (!e2e.has(file)) e2e.set(file, new Set());
+      e2e.get(file)!.add(id);
+    }
+  }
+  return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed };
+}
+
+const exercisedCache = new Map<string, ExercisedConsumers>();
+
+/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */
+export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers {
+  const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`;
+  if (!exercisedCache.has(key)) {
+    const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8'));
+    exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt }));
+  }
+  return exercisedCache.get(key)!;
+}

scripts/replay-pr-selection.ts

@@ -26,7 +26,7 @@ import { packageChangeOnlyVersion } from './test-pr-profile';
 import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data';
 import { matchGlob } from '../test/helpers/test-selection';
 import { derivedDependencies } from './pr-dependencies';
-import { PR_PROFILE_MAPS } from './test-pr-profile';
+import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile';
 
 export interface ReplayInput {
   runId: number;
@@ -59,7 +59,9 @@ export interface ReplayRow {
 /**
  * Whether a case depends on any changed file under today's dependency graph:
  * its touchfiles, the globals, and the derived reference closure
- * (scripts/pr-dependencies.ts, its global runner closure included).
+ * (scripts/pr-dependencies.ts, its global runner closure included). Like
+ * the selector, a touchfile match wins over the audited free-only exemptions,
+ * and an exempt file reaches no case through the derived closure.
  */
 export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean {
   const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES];
@@ -67,7 +69,7 @@ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): bo
   // A path absent from today's tree reaches a case through the live files that still reference it.
   const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file);
   return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern))
-    || via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)));
+    || (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))));
 }
 
 export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow {

scripts/test-health-report.ts

@@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows']
 export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string }
 export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string }
 export type IssueAction = 'upsert' | 'keep-open' | 'close';
-export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
+export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
 
 export interface HealthInputs {
   client: GhClient;
@@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
   const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
   const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled')
     .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) })));
+  const prManifests = attempt(() => need(prRuns).flatMap(run => {
+    const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate));
+    return file ? [{ run, manifest: readJson(file) }] : [];
+  }));
   const metrics: Metric[] = [];
+  let costPerMergedPr: string | undefined;
 
-  metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => {
+  metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => {
     const modes = new Map<string, number>();
-    for (const run of need(prRuns)) {
-      const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file));
-      if (!manifest) continue;
-      const mode = String(readJson(manifest).prCoverage?.mode ?? 'none');
+    for (const { manifest } of need(prManifests)) {
+      const mode = String(manifest.prCoverage?.mode ?? 'none');
       modes.set(mode, (modes.get(mode) ?? 0) + 1);
     }
     const total = [...modes.values()].reduce((a, b) => a + b, 0);
@@ -204,6 +207,51 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
       detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) };
   }));
 
+  metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => {
+    const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges));
+    if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window');
+    const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`;
+    const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort();
+    return { value: `${mean(planned)} per run (${planned.length} runs)`,
+      detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) };
+  }));
+
+  metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => {
+    const entries = need(prRecords);
+    if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window');
+    const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0);
+    const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length;
+    const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')]));
+    const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`;
+    const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort();
+    return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`,
+      detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) };
+  }));
+
+  metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => {
+    const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error);
+    if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window');
+    const records = prRecords instanceof Error ? [] : prRecords;
+    const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length;
+    const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0);
+    const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0);
+    const misses: string[] = [];
+    const flaky: string[] = [];
+    for (const { run, manifest } of shadowed) {
+      const trials = records.find(entry => entry.run.id === run.id)?.trials ?? [];
+      for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) {
+        if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue;
+        const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha
+          && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0);
+        if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`);
+      }
+    }
+    const week = 7 / sinceDays;
+    const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost';
+    return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`,
+      detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] };
+  }));
+
   metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
     const all = need(prRecords).flatMap(entry => entry.trials);
     const reused = all.filter(record => record.execution === 'reused').length;
@@ -216,6 +264,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
       .reduce((sum, record) => sum + record.cost_usd, 0);
     const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size;
     const merged = listMergedPulls(client, repo, since);
+    if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`;
     return { value: `$${spend.toFixed(2)} recorded`, detail: [
       `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`,
       `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] };
@@ -267,7 +316,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
   }));
 
   const checks = [flakeCheck(flakes), unseededCheck(root)];
-  return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) };
+  const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`;
+  return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) };
 }
 
 function listMergedPulls(client: GhClient, repo: string, since: string): number {
@@ -340,7 +390,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: (
 // --- Rendering ---
 
 export function formatHealth(report: HealthReport): string {
-  const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`];
+  const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])];
   for (const entry of report.metrics) {
     lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`);
     lines.push(`    rule: ${entry.rule}`);

scripts/test-pr-profile.ts

@@ -5,7 +5,9 @@ import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } fr
 import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection';
 import { isPaidTestFile } from '../test/helpers/paid-test-set';
 import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data';
-import { derivedDependencies, type DerivedDependencies } from './pr-dependencies';
+import {
+  checkoutExercisedConsumers, derivedDependencies, EXERCISED_MAP_PATH, type DerivedDependencies, type ExercisedConsumers,
+} from './pr-dependencies';
 
 /** Existing short behavioral probes; intersect with changed-input selection. */
 export const PR_PROFILE_CASE_IDS = [
@@ -157,6 +159,27 @@ export interface PrProfileSelection {
   missingCoverage: string[];
   needsFullValidation: boolean;
   reasons: string[];
+  /** What the exercised-path map would select (shadow mode: never changes e2e or judges). */
+  shadow?: ShadowSelection;
+}
+
+/** The exercised-path map's selection beside the real one (docs/TESTING_INTERNALS.md#exercised-path-map). */
+export interface ShadowSelection {
+  /** Gate cases the map would select. */
+  e2e: string[];
+  /** Selected gate cases the map would not select. */
+  wouldDrop: string[];
+  /** Gate cases the map would select that today's selection does not. */
+  wouldAdd: string[];
+  /** Per changed file with exercised consumers: how many gate cases it reaches, and through which edge. */
+  provenance: Array<{ file: string; summary: string }>;
+  /** Mean known cost of the dropped cases minus the added ones (the map's per-case costUsd; an unpriced case counts $0): a lower bound. */
+  predictedSavingUsd: number;
+  unpricedCases: number;
+  /** Share of observed case runs whose trials carry no known cost (PTY trials); their spend is missing from every dollar figure. */
+  unknownCostShare: number;
+  /** Why no shadow could be computed (the real selection is unaffected). */
+  error?: string;
 }
 
 /** Reject stale profile registrations and deferred cases with no scheduled home. */
@@ -216,6 +239,12 @@ export const FREE_ONLY_PR_FILES = [
   '.gstack/generated.json', // gstack's own regen registry: gstack-regen/gstack-pregate read it from origin/<base> of the repo they run in; paid cases run in fixture repos.
   '.gstack/pregate.json', // gstack's own pre-gate policy: same trust boundary, same reason.
   '.gstack/ship-policy.json', // gstack's own ship policy: gstack-ship-policy/gstack-restamp read it from origin/<base> of the repo they run in; same reason.
+  '.gstack/ship-policy.example.json', // The example beside it: only .gitignore and docs/ship-policy.md name it.
+  '.github/dependabot.yml', // Dependabot's own config: read by GitHub, and by free test/compare-pr-version.test.ts; dependabot runs skip the paid lane.
+  '.osv-scanner.toml', // Read by .github/workflows/osv-scanner.yml and the free OSV tests (test/osv-*.test.ts).
+  'setup.ps1', // The Windows installer: run only by .github/workflows/windows-setup-e2e.yml; the paid lane is Linux.
+  '.gitlab-ci.yml', // gstack's own GitLab config: skills probe the fixture repo's .gitlab-ci.yml, never this one.
+  '.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts.
   'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it.
   'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it.
   // Reporting and launch tools: they read CI history or start a lane, never run inside a paid case.
@@ -230,6 +259,7 @@ export const FREE_ONLY_PR_FILES = [
   '.github/workflows/cso-runtime-qualification.yml',
   '.github/workflows/cso-scanner-images.yml',
   '.github/workflows/dependency-review.yml',
+  '.github/workflows/free-durations-record.yml',
   '.github/workflows/free-tests.yml',
   '.github/workflows/make-pdf-gate.yml',
   '.github/workflows/model-policy-freshness.yml',
@@ -273,9 +303,9 @@ export const PR_FULL_GATE_FILES = [
 export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const;
 
 /** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */
-export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const;
+export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/', 'capy/'] as const;
 
-function knownNonBehaviorFile(file: string): boolean {
+export function knownNonBehaviorFile(file: string): boolean {
   // A mapped dependency still wins over these exemptions. New helper,
   // runtime, dependency, or paid-workflow files are deliberately not exempted.
   return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file)
@@ -290,9 +320,13 @@ function knownNonBehaviorFile(file: string): boolean {
 }
 
 export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback';
+export const EXERCISED_MAP_ANCHOR = 'docs/TESTING_INTERNALS.md#exercised-path-map';
 
 /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */
 export function unknownFileLabel(file: string): { label: string; fix: string } {
+  if (matches(file, PR_FULL_GATE_FILES)) {
+    return { label: 'full-gate input', fix: `${file} is in PR_FULL_GATE_FILES (scripts/test-pr-profile.ts): every paid case consumes it, so the full gate is intended and no mapping narrows it` };
+  }
   if (isPaidTestFile(file) || file.startsWith('test/helpers/')) {
     return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` };
   }
@@ -338,6 +372,8 @@ export function selectPrProfile(options: {
   readSource?: (file: string) => string;
   /** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */
   derived?: DerivedDependencies | null;
+  /** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */
+  exercised?: ExercisedConsumers | null;
 }): PrProfileSelection {
   const maps = options.maps ?? PR_PROFILE_MAPS;
   const profile = options.profile ?? PR_PROFILE_CASE_IDS;
@@ -351,7 +387,7 @@ export function selectPrProfile(options: {
     ...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(),
     ...maps.globalTouchfiles,
   ])];
-  const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH
+  const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH && file !== EXERCISED_MAP_PATH
     && !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file));
   // Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate.
   const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null;
@@ -380,6 +416,12 @@ export function selectPrProfile(options: {
     if (found.e2e.size + found.judges.size > 0) consumers.set(file, found);
     else noConsumerFiles.push(file);
   }
+  // A failed import scan leaves some closure unknown: every unmapped file restores the full gate (fail closed).
+  const scanFailed = derived && unmapped.length ? [...derived.scanErrors.keys()].sort() : [];
+  if (scanFailed.length) {
+    consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0;
+    for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` });
+  }
   const derivedFiles = [...consumers.keys()];
   const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file));
   const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES));
@@ -422,25 +464,80 @@ export function selectPrProfile(options: {
   if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`);
   if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`);
   else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
+  if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`);
   if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`);
   if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`);
   if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`);
   if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`);
   if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`);
   if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`);
+  const exercised = options.exercised !== undefined ? options.exercised : maps === PR_PROFILE_MAPS ? checkoutExercised() : null;
+  const shadow = exercised instanceof Error ? emptyShadow(e2e, exercised.message)
+    : exercised ? shadowSelection({ ...options, maps, profile }, { e2e, fallback, derived, exercised, files }) : undefined;
   return {
     mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred,
     unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases,
     derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles,
-    deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons,
+    deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}),
   };
 }
 
+function checkoutExercised(): ExercisedConsumers | Error {
+  try { return checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); }
+  catch (error) { return error instanceof Error ? error : new Error(String(error)); }
+}
+
+function emptyShadow(e2e: string[], error: string): ShadowSelection {
+  return { e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error };
+}
+
+/**
+ * Shadow selection (variant cu of the measured prototype): the same policy
+ * with the exercised map in place of the derived closure, unioned with the
+ * exercised consumers of every changed file that is not global or full-gate.
+ * A full-gate fallback stays full.
+ */
+function shadowSelection(options: Parameters<typeof selectPrProfile>[0] & { maps: PrProfileMaps },
+  today: { e2e: string[]; fallback: boolean; derived: DerivedDependencies | null; exercised: ExercisedConsumers; files: string[] }): ShadowSelection {
+  const { maps } = options;
+  const { exercised } = today;
+  const observedCases = Object.values(exercised.map?.cases ?? {});
+  const observations = observedCases.reduce((sum, entry) => sum + entry.observations, 0);
+  const unknownCostShare = observations ? Math.round(1000 * (1 - observedCases.reduce((sum, entry) => sum + (entry.costRuns ?? 0), 0) / observations)) / 1000 : 0;
+  if (today.fallback) return { e2e: today.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare };
+  const gate = (ids: Iterable<string>) => [...ids].filter(id => maps.tiers[id] === 'gate');
+  const reach = new Map<string, string[]>();
+  const inner = selectPrProfile({ ...options, derived: today.derived ? { ...today.derived, e2e: exercised.e2e } : null, exercised: null });
+  const sources = [...new Set([...today.files, ...Object.values(options.sourceAliases ?? {})])];
+  const shadowSet = new Set(inner.e2e);
+  for (const file of sources) {
+    if (today.derived?.global.has(file) || matches(file, PR_FULL_GATE_FILES) || knownNonBehaviorFile(file)) continue;
+    const via = !today.derived || today.derived.tracked.has(file) ? [file] : today.derived.referencers(file);
+    const ids = gate(new Set(via.flatMap(source => [...exercised.e2e.get(source) ?? []])));
+    if (ids.length) reach.set(file, ids);
+    for (const id of ids) shadowSet.add(id);
+  }
+  const e2e = [...shadowSet].sort();
+  const wouldDrop = today.e2e.filter(id => !shadowSet.has(id));
+  const wouldAdd = e2e.filter(id => !today.e2e.includes(id));
+  const cost = (ids: string[]) => ids.reduce((sum, id) => sum + (exercised.map?.cases[id]?.costUsd ?? 0), 0);
+  const provenance = [...reach].map(([file, ids]) => {
+    const groups = new Map<string, number>();
+    for (const id of ids) {
+      const label = exercised.closure(id).get(file) ?? 'referencer';
+      groups.set(label, (groups.get(label) ?? 0) + 1);
+    }
+    return { file, summary: `selects ${ids.length} case(s): ${[...groups].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([label, n]) => `${label} (${n})`).join(', ')}` };
+  });
+  return { e2e, wouldDrop, wouldAdd, provenance, predictedSavingUsd: Math.round((cost(wouldDrop) - cost(wouldAdd)) * 100) / 100,
+    unpricedCases: [...wouldDrop, ...wouldAdd].filter(id => exercised.map?.cases[id]?.costUsd === undefined).length, unknownCostShare };
+}
+
 /** The manifest fields the coverage summary reads (a PaidRunManifest subset). */
 export interface CoverageSummaryInput {
   profile?: string;
   selection?: { e2e: string[] | null; judges: string[] | null };
-  prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles'>>;
+  prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles' | 'shadow'>>;
 }
 
 /**
@@ -461,6 +558,9 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?:
     lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`);
     for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(`  - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`);
   }
+  const shadow = coverage?.shadow;
+  if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected`
+    : `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`);
   if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`);
   if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) {
     lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`);