evals: map-diff selection — a touchfiles-data edit runs only what changed

Editing the eval dep-list data no longer forces the full ~$38 / 30-45min suite (measured on 21.9% of recent commits). When touchfiles-data.ts is in the diff, selection now evaluates the BASE version (git show -> mkdtemp -> spawnSync bun child printing the four maps as JSON — sync because e2e-helpers selects at module scope) and JSON-diffs per key: added entries, edited dep lists, and tier flips are selected; keys removed from all maps are reported, never silently dropped; a GLOBAL_TOUCHFILES edit still runs everything. FAIL-CLOSED with named causes: missing-base-ref, git-show-failed, import-failed, shape-mismatch each degrade to run-all and print 'selection: global — touchfiles-data changed (<cause>)' (D9 — silently expensive beats silently wrong, but never silently). eval:select prints 'selected N of M, reason: ...' + removed tests; --base scopes the map-diff too. The temporary conservative GLOBAL entry for touchfiles-data.ts is gone — its changes route through the map-diff. 23 new free tests: pure-core fixtures, selectTests wiring incl. a poison-injection guard, and a temp git repo exercising every fail-closed cause end-to-end. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-21 21:47:32 +02:00 · 2026-08-15 08:30:09 -07:00
parent f23b2263bc
commit 820afa26e8
5 changed files with 646 additions and 17 deletions
@@ -6,13 +6,36 @@
 * checks `git diff` and only runs tests whose dependencies were modified.
 * Override with EVALS_ALL=1 to run everything.
 *
+ * When touchfiles-data.ts itself changed, selection uses MAP-DIFF instead of
+ * a global run-all: the old version of the data file is loaded from git and
+ * evaluated in a bun child process (the literal-only tripwire in
+ * test/touchfiles-facade.test.ts bounds what executing it can do), the four
+ * maps are diffed per key, and only tests whose entry was added, whose
+ * dep-list changed, or whose tier flipped are selected. Any failure on that
+ * path fails CLOSED: run all tests, with the cause in the reason string.
+ *
+ * Everything here is synchronous by design: e2e-helpers.ts and the *-e2e
+ * test files compute selection at module load, so the old-file evaluation
+ * happens in a spawnSync'd bun child rather than a dynamic import.
+ *
 * Import sites should keep using the ./touchfiles facade, which re-exports
 * both this module and the data module.
 */

 import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';

-import { GLOBAL_TOUCHFILES } from './touchfiles-data';
+import {
+  E2E_TOUCHFILES,
+  E2E_TIERS,
+  LLM_JUDGE_TOUCHFILES,
+  GLOBAL_TOUCHFILES,
+} from './touchfiles-data';
+
+/** Repo-relative path of the pure-data file (the map-diff subject). */
+export const TOUCHFILES_DATA_PATH = 'test/helpers/touchfiles-data.ts';

 // --- Glob matching ---

@@ -58,37 +81,267 @@ export function getChangedFiles(baseBranch: string, cwd: string): string[] {
  return result.stdout.toString().trim().split('\n').filter(Boolean);
 }

+// --- Touchfile map diffing ---
+
+/** The four exports of touchfiles-data.ts, as plain data. */
+export interface TouchfileMaps {
+  E2E_TOUCHFILES: Record<string, string[]>;
+  E2E_TIERS: Record<string, string>;
+  LLM_JUDGE_TOUCHFILES: Record<string, string[]>;
+  GLOBAL_TOUCHFILES: string[];
+}
+
+export type MapDiffCause =
+  | 'missing-base-ref'
+  | 'git-show-failed'
+  | 'import-failed'
+  | 'shape-mismatch';
+
+export type MapDiffOutcome =
+  | {
+      ok: true;
+      /** Tests whose entry was added, dep-list changed, or tier flipped. */
+      changedTests: string[];
+      /** Keys present in the old maps but gone from every new map (reported, not selected). */
+      removedTests: string[];
+      /** True when the GLOBAL_TOUCHFILES set itself changed — not attributable to any test. */
+      globalTouchfilesChanged: boolean;
+    }
+  | { ok: false; cause: MapDiffCause };
+
+/** Current maps as a TouchfileMaps value (the "new" side of the diff). */
+const CURRENT_MAPS: TouchfileMaps = {
+  E2E_TOUCHFILES,
+  E2E_TIERS,
+  LLM_JUDGE_TOUCHFILES,
+  GLOBAL_TOUCHFILES,
+};
+
+function isStringArray(v: unknown): v is string[] {
+  return Array.isArray(v) && v.every(x => typeof x === 'string');
+}
+
+function isRecordOfStringArrays(v: unknown): v is Record<string, string[]> {
+  return !!v && typeof v === 'object' && !Array.isArray(v)
+    && Object.values(v).every(isStringArray);
+}
+
+function isRecordOfStrings(v: unknown): v is Record<string, string> {
+  return !!v && typeof v === 'object' && !Array.isArray(v)
+    && Object.values(v).every(x => typeof x === 'string');
+}
+
+function isTouchfileMaps(v: unknown): v is TouchfileMaps {
+  if (!v || typeof v !== 'object') return false;
+  const o = v as Record<string, unknown>;
+  return isRecordOfStringArrays(o.E2E_TOUCHFILES)
+    && isRecordOfStrings(o.E2E_TIERS)
+    && isRecordOfStringArrays(o.LLM_JUDGE_TOUCHFILES)
+    && isStringArray(o.GLOBAL_TOUCHFILES);
+}
+
+/**
+ * Pure map-diff core (injectable for tests — no git, no filesystem).
+ *
+ * A key counts as CHANGED when it was added to any per-key map, its dep-list
+ * array differs, or its tier value flipped. A key counts as REMOVED only when
+ * it is gone from every new per-key map; a key dropped from one map but still
+ * present in another (e.g. tier entry deleted, touchfile entry kept) counts
+ * as changed — conservative, because the test still exists with a different
+ * configuration. GLOBAL_TOUCHFILES is compared as a set; a change there is
+ * not attributable to any test and is flagged for the caller to treat as
+ * "run all".
+ */
+export function diffTouchfileMapsCore(
+  oldMaps: TouchfileMaps,
+  newMaps: TouchfileMaps,
+): { changedTests: string[]; removedTests: string[]; globalTouchfilesChanged: boolean } {
+  const perKeyMapNames = ['E2E_TOUCHFILES', 'E2E_TIERS', 'LLM_JUDGE_TOUCHFILES'] as const;
+  const changed = new Set<string>();
+  const rawRemoved = new Set<string>();
+
+  for (const mapName of perKeyMapNames) {
+    const oldMap: Record<string, unknown> = oldMaps[mapName] ?? {};
+    const newMap: Record<string, unknown> = newMaps[mapName] ?? {};
+    for (const key of Object.keys(newMap)) {
+      if (!(key in oldMap)) {
+        changed.add(key); // added
+      } else if (JSON.stringify(oldMap[key]) !== JSON.stringify(newMap[key])) {
+        changed.add(key); // dep-list edited or tier flipped
+      }
+    }
+    for (const key of Object.keys(oldMap)) {
+      if (!(key in newMap)) rawRemoved.add(key);
+    }
+  }
+
+  const removed = new Set<string>();
+  for (const key of rawRemoved) {
+    const stillExists = perKeyMapNames.some(m => key in (newMaps[m] ?? {}));
+    if (stillExists) changed.add(key);
+    else removed.add(key);
+  }
+
+  const sortedSet = (arr: string[]) => JSON.stringify([...arr].sort());
+  const globalTouchfilesChanged =
+    sortedSet(oldMaps.GLOBAL_TOUCHFILES ?? []) !== sortedSet(newMaps.GLOBAL_TOUCHFILES ?? []);
+
+  return {
+    changedTests: [...changed].sort(),
+    removedTests: [...removed].sort(),
+    globalTouchfilesChanged,
+  };
+}
+
+/**
+ * Load the OLD touchfiles-data.ts from git and diff it against the current
+ * maps. Synchronous: the old file is written to a temp dir and evaluated in
+ * a spawnSync'd bun child that prints the four maps as JSON (module-scope
+ * callers like e2e-helpers.ts cannot await).
+ *
+ * FAIL-CLOSED: every failure returns `{ ok: false, cause }` and the caller
+ * must treat that as "data change is global — run all tests".
+ *
+ * `newMaps` is injectable so integration tests can diff a temp repo's old
+ * version against a fixture instead of this repo's live maps.
+ */
+export function diffTouchfileMaps(
+  baseRef: string,
+  cwd: string,
+  newMaps: TouchfileMaps = CURRENT_MAPS,
+): MapDiffOutcome {
+  try {
+    const verify = spawnSync('git', ['rev-parse', '--verify', baseRef], {
+      cwd, stdio: 'pipe', timeout: 3000,
+    });
+    if (verify.status !== 0) return { ok: false, cause: 'missing-base-ref' };
+
+    const show = spawnSync('git', ['show', `${baseRef}:${TOUCHFILES_DATA_PATH}`], {
+      cwd, stdio: 'pipe', timeout: 5000, maxBuffer: 8 * 1024 * 1024,
+    });
+    if (show.status !== 0) return { ok: false, cause: 'git-show-failed' };
+    const oldSource = show.stdout.toString();
+
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-'));
+    try {
+      const dataPath = path.join(tempDir, 'touchfiles-data.ts');
+      fs.writeFileSync(dataPath, oldSource);
+      const loaderPath = path.join(tempDir, 'load-maps.ts');
+      fs.writeFileSync(loaderPath, [
+        `const m = await import(${JSON.stringify(dataPath)});`,
+        'console.log(JSON.stringify({',
+        '  E2E_TOUCHFILES: m.E2E_TOUCHFILES,',
+        '  E2E_TIERS: m.E2E_TIERS,',
+        '  LLM_JUDGE_TOUCHFILES: m.LLM_JUDGE_TOUCHFILES,',
+        '  GLOBAL_TOUCHFILES: m.GLOBAL_TOUCHFILES,',
+        '}));',
+        '',
+      ].join('\n'));
+
+      // process.execPath is the bun binary when running under bun.
+      const run = spawnSync(process.execPath, ['run', loaderPath], {
+        stdio: 'pipe', timeout: 20000, maxBuffer: 8 * 1024 * 1024,
+      });
+      if (run.status !== 0) return { ok: false, cause: 'import-failed' };
+
+      let oldMaps: unknown;
+      try {
+        oldMaps = JSON.parse(run.stdout.toString());
+      } catch {
+        return { ok: false, cause: 'import-failed' };
+      }
+      if (!isTouchfileMaps(oldMaps)) return { ok: false, cause: 'shape-mismatch' };
+
+      return { ok: true, ...diffTouchfileMapsCore(oldMaps, newMaps) };
+    } finally {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  } catch {
+    // Unexpected failure anywhere in the pipeline (temp dir, git, child
+    // process) — same fail-closed contract as an evaluation failure.
+    return { ok: false, cause: 'import-failed' };
+  }
+}
+
 // --- Test selection ---

 /**
 * Select tests to run based on changed files.
 *
 * Algorithm:
- * 1. If any changed file matches a global touchfile → run ALL tests
- * 2. Otherwise, for each test, check if any changed file matches its patterns
- * 3. Return selected + skipped lists with reason
+ * 1. If any changed file (other than touchfiles-data.ts) matches a global
+ *    touchfile → run ALL tests
+ * 2. If touchfiles-data.ts changed → map-diff it against the base ref and
+ *    select only the tests whose map entries changed (fail-closed: any
+ *    map-diff failure runs ALL tests, with the cause in the reason string)
+ * 3. For each test, check if any other changed file matches its patterns
+ * 4. Return selected + skipped lists with reason (union of 2 and 3)
+ *
+ * `opts.baseRef` / `opts.cwd` scope the map-diff; they default to
+ * EVALS_BASE || detectBaseBranch || 'main' and the repo root — the same
+ * resolution the module-scope callers (e2e-helpers.ts et al.) use to compute
+ * `changedFiles`, so the two sides of the diff stay consistent.
+ * `opts.mapDiff` injects a precomputed outcome (for tests).
 */
 export function selectTests(
  changedFiles: string[],
  touchfiles: Record<string, string[]>,
  globalTouchfiles: string[] = GLOBAL_TOUCHFILES,
-): { selected: string[]; skipped: string[]; reason: string } {
+  opts: { baseRef?: string; cwd?: string; mapDiff?: MapDiffOutcome } = {},
+): { selected: string[]; skipped: string[]; reason: string; removedTests?: string[] } {
  const allTestNames = Object.keys(touchfiles);
+  const dataChanged = changedFiles.includes(TOUCHFILES_DATA_PATH);

-  // Global touchfile hit → run all
+  // Global touchfile hit → run all. touchfiles-data.ts is excluded here —
+  // its changes route through map-diff below instead of a global run-all.
  for (const file of changedFiles) {
+    if (file === TOUCHFILES_DATA_PATH) continue;
    if (globalTouchfiles.some(g => matchGlob(file, g))) {
      return { selected: allTestNames, skipped: [], reason: `global: ${file}` };
    }
  }

-  // Per-test matching
+  // Map-diff path for data-file changes
+  let mapDiffSelected: Set<string> | null = null;
+  let removedTests: string[] | undefined;
+  if (dataChanged) {
+    const cwd = opts.cwd ?? path.resolve(import.meta.dir, '..', '..');
+    const baseRef = opts.baseRef
+      || process.env.EVALS_BASE
+      || detectBaseBranch(cwd)
+      || 'main';
+    const outcome = opts.mapDiff ?? diffTouchfileMaps(baseRef, cwd);
+    if (!outcome.ok) {
+      return {
+        selected: allTestNames,
+        skipped: [],
+        reason: `global — touchfiles-data changed (${outcome.cause})`,
+      };
+    }
+    if (outcome.globalTouchfilesChanged) {
+      return {
+        selected: allTestNames,
+        skipped: [],
+        reason: 'global — touchfiles-data changed (GLOBAL_TOUCHFILES edited)',
+      };
+    }
+    // Scope to this map's keys (E2E and LLM-judge selections run separately).
+    mapDiffSelected = new Set(outcome.changedTests.filter(t => t in touchfiles));
+    removedTests = outcome.removedTests;
+  }
+
+  // Per-test matching for the remaining changed files
+  const otherFiles = changedFiles.filter(f => f !== TOUCHFILES_DATA_PATH);
  const selected: string[] = [];
  const skipped: string[] = [];
  for (const [testName, patterns] of Object.entries(touchfiles)) {
-    const hit = changedFiles.some(f => patterns.some(p => matchGlob(f, p)));
+    const hit = otherFiles.some(f => patterns.some(p => matchGlob(f, p)))
+      || (mapDiffSelected !== null && mapDiffSelected.has(testName));
    (hit ? selected : skipped).push(testName);
  }

+  if (dataChanged) {
+    return { selected, skipped, reason: 'map-diff', removedTests };
+  }
  return { selected, skipped, reason: 'diff' };
 }