From 820afa26e8e0f56b3b59de40bc60a6f37b79ab85 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 08:30:09 -0700 Subject: [PATCH] =?UTF-8?q?evals:=20map-diff=20selection=20=E2=80=94=20a?= =?UTF-8?q?=20touchfiles-data=20edit=20runs=20only=20what=20changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editing the eval dep-list data no longer forces the full ~$38 / 30-45min suite (measured on 21.9% of recent commits). When touchfiles-data.ts is in the diff, selection now evaluates the BASE version (git show -> mkdtemp -> spawnSync bun child printing the four maps as JSON — sync because e2e-helpers selects at module scope) and JSON-diffs per key: added entries, edited dep lists, and tier flips are selected; keys removed from all maps are reported, never silently dropped; a GLOBAL_TOUCHFILES edit still runs everything. FAIL-CLOSED with named causes: missing-base-ref, git-show-failed, import-failed, shape-mismatch each degrade to run-all and print 'selection: global — touchfiles-data changed ()' (D9 — silently expensive beats silently wrong, but never silently). eval:select prints 'selected N of M, reason: ...' + removed tests; --base scopes the map-diff too. The temporary conservative GLOBAL entry for touchfiles-data.ts is gone — its changes route through the map-diff. 23 new free tests: pure-core fixtures, selectTests wiring incl. a poison-injection guard, and a temp git repo exercising every fail-closed cause end-to-end. Co-Authored-By: Claude Fable 5 --- scripts/eval-select.ts | 15 +- test/helpers/test-selection.ts | 269 ++++++++++++++++++++++- test/helpers/touchfiles-data.ts | 10 +- test/helpers/touchfiles.ts | 9 + test/touchfiles-map-diff.test.ts | 360 +++++++++++++++++++++++++++++++ 5 files changed, 646 insertions(+), 17 deletions(-) create mode 100644 test/touchfiles-map-diff.test.ts diff --git a/scripts/eval-select.ts b/scripts/eval-select.ts index cdbdcc848..97476b720 100644 --- a/scripts/eval-select.ts +++ b/scripts/eval-select.ts @@ -38,8 +38,11 @@ if (changedFiles.length === 0) { process.exit(0); } -const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES); -const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES); +// baseRef/cwd scope the map-diff path (used when touchfiles-data.ts changed) +// to the same base this script diffed against — including a --base override. +const selectOpts = { baseRef: baseBranch, cwd: ROOT }; +const e2eSelection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts); +const llmSelection = selectTests(changedFiles, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES, selectOpts); if (jsonMode) { console.log(JSON.stringify({ @@ -49,6 +52,7 @@ if (jsonMode) { selected: e2eSelection.selected, skipped: e2eSelection.skipped, reason: e2eSelection.reason, + removed_tests: e2eSelection.removedTests ?? [], count: `${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length}`, }, llm_judge: { @@ -63,7 +67,10 @@ if (jsonMode) { console.log(`Changed files: ${changedFiles.length}`); console.log(); - console.log(`E2E (${e2eSelection.reason}): ${e2eSelection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests`); + console.log(`E2E: selected ${e2eSelection.selected.length} of ${Object.keys(E2E_TOUCHFILES).length}, reason: ${e2eSelection.reason}`); + if (e2eSelection.removedTests && e2eSelection.removedTests.length > 0) { + console.log(` Removed from maps (reported, not selected): ${e2eSelection.removedTests.join(', ')}`); + } if (e2eSelection.selected.length > 0 && e2eSelection.selected.length < Object.keys(E2E_TOUCHFILES).length) { console.log(` Selected: ${e2eSelection.selected.join(', ')}`); console.log(` Skipped: ${e2eSelection.skipped.join(', ')}`); @@ -74,7 +81,7 @@ if (jsonMode) { } console.log(); - console.log(`LLM-judge (${llmSelection.reason}): ${llmSelection.selected.length}/${Object.keys(LLM_JUDGE_TOUCHFILES).length} tests`); + console.log(`LLM-judge: selected ${llmSelection.selected.length} of ${Object.keys(LLM_JUDGE_TOUCHFILES).length}, reason: ${llmSelection.reason}`); if (llmSelection.selected.length > 0 && llmSelection.selected.length < Object.keys(LLM_JUDGE_TOUCHFILES).length) { console.log(` Selected: ${llmSelection.selected.join(', ')}`); console.log(` Skipped: ${llmSelection.skipped.join(', ')}`); diff --git a/test/helpers/test-selection.ts b/test/helpers/test-selection.ts index feb09e76b..4037c49d6 100644 --- a/test/helpers/test-selection.ts +++ b/test/helpers/test-selection.ts @@ -6,13 +6,36 @@ * checks `git diff` and only runs tests whose dependencies were modified. * Override with EVALS_ALL=1 to run everything. * + * When touchfiles-data.ts itself changed, selection uses MAP-DIFF instead of + * a global run-all: the old version of the data file is loaded from git and + * evaluated in a bun child process (the literal-only tripwire in + * test/touchfiles-facade.test.ts bounds what executing it can do), the four + * maps are diffed per key, and only tests whose entry was added, whose + * dep-list changed, or whose tier flipped are selected. Any failure on that + * path fails CLOSED: run all tests, with the cause in the reason string. + * + * Everything here is synchronous by design: e2e-helpers.ts and the *-e2e + * test files compute selection at module load, so the old-file evaluation + * happens in a spawnSync'd bun child rather than a dynamic import. + * * Import sites should keep using the ./touchfiles facade, which re-exports * both this module and the data module. */ import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; -import { GLOBAL_TOUCHFILES } from './touchfiles-data'; +import { + E2E_TOUCHFILES, + E2E_TIERS, + LLM_JUDGE_TOUCHFILES, + GLOBAL_TOUCHFILES, +} from './touchfiles-data'; + +/** Repo-relative path of the pure-data file (the map-diff subject). */ +export const TOUCHFILES_DATA_PATH = 'test/helpers/touchfiles-data.ts'; // --- Glob matching --- @@ -58,37 +81,267 @@ export function getChangedFiles(baseBranch: string, cwd: string): string[] { return result.stdout.toString().trim().split('\n').filter(Boolean); } +// --- Touchfile map diffing --- + +/** The four exports of touchfiles-data.ts, as plain data. */ +export interface TouchfileMaps { + E2E_TOUCHFILES: Record; + E2E_TIERS: Record; + LLM_JUDGE_TOUCHFILES: Record; + GLOBAL_TOUCHFILES: string[]; +} + +export type MapDiffCause = + | 'missing-base-ref' + | 'git-show-failed' + | 'import-failed' + | 'shape-mismatch'; + +export type MapDiffOutcome = + | { + ok: true; + /** Tests whose entry was added, dep-list changed, or tier flipped. */ + changedTests: string[]; + /** Keys present in the old maps but gone from every new map (reported, not selected). */ + removedTests: string[]; + /** True when the GLOBAL_TOUCHFILES set itself changed — not attributable to any test. */ + globalTouchfilesChanged: boolean; + } + | { ok: false; cause: MapDiffCause }; + +/** Current maps as a TouchfileMaps value (the "new" side of the diff). */ +const CURRENT_MAPS: TouchfileMaps = { + E2E_TOUCHFILES, + E2E_TIERS, + LLM_JUDGE_TOUCHFILES, + GLOBAL_TOUCHFILES, +}; + +function isStringArray(v: unknown): v is string[] { + return Array.isArray(v) && v.every(x => typeof x === 'string'); +} + +function isRecordOfStringArrays(v: unknown): v is Record { + return !!v && typeof v === 'object' && !Array.isArray(v) + && Object.values(v).every(isStringArray); +} + +function isRecordOfStrings(v: unknown): v is Record { + return !!v && typeof v === 'object' && !Array.isArray(v) + && Object.values(v).every(x => typeof x === 'string'); +} + +function isTouchfileMaps(v: unknown): v is TouchfileMaps { + if (!v || typeof v !== 'object') return false; + const o = v as Record; + return isRecordOfStringArrays(o.E2E_TOUCHFILES) + && isRecordOfStrings(o.E2E_TIERS) + && isRecordOfStringArrays(o.LLM_JUDGE_TOUCHFILES) + && isStringArray(o.GLOBAL_TOUCHFILES); +} + +/** + * Pure map-diff core (injectable for tests — no git, no filesystem). + * + * A key counts as CHANGED when it was added to any per-key map, its dep-list + * array differs, or its tier value flipped. A key counts as REMOVED only when + * it is gone from every new per-key map; a key dropped from one map but still + * present in another (e.g. tier entry deleted, touchfile entry kept) counts + * as changed — conservative, because the test still exists with a different + * configuration. GLOBAL_TOUCHFILES is compared as a set; a change there is + * not attributable to any test and is flagged for the caller to treat as + * "run all". + */ +export function diffTouchfileMapsCore( + oldMaps: TouchfileMaps, + newMaps: TouchfileMaps, +): { changedTests: string[]; removedTests: string[]; globalTouchfilesChanged: boolean } { + const perKeyMapNames = ['E2E_TOUCHFILES', 'E2E_TIERS', 'LLM_JUDGE_TOUCHFILES'] as const; + const changed = new Set(); + const rawRemoved = new Set(); + + for (const mapName of perKeyMapNames) { + const oldMap: Record = oldMaps[mapName] ?? {}; + const newMap: Record = newMaps[mapName] ?? {}; + for (const key of Object.keys(newMap)) { + if (!(key in oldMap)) { + changed.add(key); // added + } else if (JSON.stringify(oldMap[key]) !== JSON.stringify(newMap[key])) { + changed.add(key); // dep-list edited or tier flipped + } + } + for (const key of Object.keys(oldMap)) { + if (!(key in newMap)) rawRemoved.add(key); + } + } + + const removed = new Set(); + for (const key of rawRemoved) { + const stillExists = perKeyMapNames.some(m => key in (newMaps[m] ?? {})); + if (stillExists) changed.add(key); + else removed.add(key); + } + + const sortedSet = (arr: string[]) => JSON.stringify([...arr].sort()); + const globalTouchfilesChanged = + sortedSet(oldMaps.GLOBAL_TOUCHFILES ?? []) !== sortedSet(newMaps.GLOBAL_TOUCHFILES ?? []); + + return { + changedTests: [...changed].sort(), + removedTests: [...removed].sort(), + globalTouchfilesChanged, + }; +} + +/** + * Load the OLD touchfiles-data.ts from git and diff it against the current + * maps. Synchronous: the old file is written to a temp dir and evaluated in + * a spawnSync'd bun child that prints the four maps as JSON (module-scope + * callers like e2e-helpers.ts cannot await). + * + * FAIL-CLOSED: every failure returns `{ ok: false, cause }` and the caller + * must treat that as "data change is global — run all tests". + * + * `newMaps` is injectable so integration tests can diff a temp repo's old + * version against a fixture instead of this repo's live maps. + */ +export function diffTouchfileMaps( + baseRef: string, + cwd: string, + newMaps: TouchfileMaps = CURRENT_MAPS, +): MapDiffOutcome { + try { + const verify = spawnSync('git', ['rev-parse', '--verify', baseRef], { + cwd, stdio: 'pipe', timeout: 3000, + }); + if (verify.status !== 0) return { ok: false, cause: 'missing-base-ref' }; + + const show = spawnSync('git', ['show', `${baseRef}:${TOUCHFILES_DATA_PATH}`], { + cwd, stdio: 'pipe', timeout: 5000, maxBuffer: 8 * 1024 * 1024, + }); + if (show.status !== 0) return { ok: false, cause: 'git-show-failed' }; + const oldSource = show.stdout.toString(); + + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-')); + try { + const dataPath = path.join(tempDir, 'touchfiles-data.ts'); + fs.writeFileSync(dataPath, oldSource); + const loaderPath = path.join(tempDir, 'load-maps.ts'); + fs.writeFileSync(loaderPath, [ + `const m = await import(${JSON.stringify(dataPath)});`, + 'console.log(JSON.stringify({', + ' E2E_TOUCHFILES: m.E2E_TOUCHFILES,', + ' E2E_TIERS: m.E2E_TIERS,', + ' LLM_JUDGE_TOUCHFILES: m.LLM_JUDGE_TOUCHFILES,', + ' GLOBAL_TOUCHFILES: m.GLOBAL_TOUCHFILES,', + '}));', + '', + ].join('\n')); + + // process.execPath is the bun binary when running under bun. + const run = spawnSync(process.execPath, ['run', loaderPath], { + stdio: 'pipe', timeout: 20000, maxBuffer: 8 * 1024 * 1024, + }); + if (run.status !== 0) return { ok: false, cause: 'import-failed' }; + + let oldMaps: unknown; + try { + oldMaps = JSON.parse(run.stdout.toString()); + } catch { + return { ok: false, cause: 'import-failed' }; + } + if (!isTouchfileMaps(oldMaps)) return { ok: false, cause: 'shape-mismatch' }; + + return { ok: true, ...diffTouchfileMapsCore(oldMaps, newMaps) }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + } catch { + // Unexpected failure anywhere in the pipeline (temp dir, git, child + // process) — same fail-closed contract as an evaluation failure. + return { ok: false, cause: 'import-failed' }; + } +} + // --- Test selection --- /** * Select tests to run based on changed files. * * Algorithm: - * 1. If any changed file matches a global touchfile → run ALL tests - * 2. Otherwise, for each test, check if any changed file matches its patterns - * 3. Return selected + skipped lists with reason + * 1. If any changed file (other than touchfiles-data.ts) matches a global + * touchfile → run ALL tests + * 2. If touchfiles-data.ts changed → map-diff it against the base ref and + * select only the tests whose map entries changed (fail-closed: any + * map-diff failure runs ALL tests, with the cause in the reason string) + * 3. For each test, check if any other changed file matches its patterns + * 4. Return selected + skipped lists with reason (union of 2 and 3) + * + * `opts.baseRef` / `opts.cwd` scope the map-diff; they default to + * EVALS_BASE || detectBaseBranch || 'main' and the repo root — the same + * resolution the module-scope callers (e2e-helpers.ts et al.) use to compute + * `changedFiles`, so the two sides of the diff stay consistent. + * `opts.mapDiff` injects a precomputed outcome (for tests). */ export function selectTests( changedFiles: string[], touchfiles: Record, globalTouchfiles: string[] = GLOBAL_TOUCHFILES, -): { selected: string[]; skipped: string[]; reason: string } { + opts: { baseRef?: string; cwd?: string; mapDiff?: MapDiffOutcome } = {}, +): { selected: string[]; skipped: string[]; reason: string; removedTests?: string[] } { const allTestNames = Object.keys(touchfiles); + const dataChanged = changedFiles.includes(TOUCHFILES_DATA_PATH); - // Global touchfile hit → run all + // Global touchfile hit → run all. touchfiles-data.ts is excluded here — + // its changes route through map-diff below instead of a global run-all. for (const file of changedFiles) { + if (file === TOUCHFILES_DATA_PATH) continue; if (globalTouchfiles.some(g => matchGlob(file, g))) { return { selected: allTestNames, skipped: [], reason: `global: ${file}` }; } } - // Per-test matching + // Map-diff path for data-file changes + let mapDiffSelected: Set | null = null; + let removedTests: string[] | undefined; + if (dataChanged) { + const cwd = opts.cwd ?? path.resolve(import.meta.dir, '..', '..'); + const baseRef = opts.baseRef + || process.env.EVALS_BASE + || detectBaseBranch(cwd) + || 'main'; + const outcome = opts.mapDiff ?? diffTouchfileMaps(baseRef, cwd); + if (!outcome.ok) { + return { + selected: allTestNames, + skipped: [], + reason: `global — touchfiles-data changed (${outcome.cause})`, + }; + } + if (outcome.globalTouchfilesChanged) { + return { + selected: allTestNames, + skipped: [], + reason: 'global — touchfiles-data changed (GLOBAL_TOUCHFILES edited)', + }; + } + // Scope to this map's keys (E2E and LLM-judge selections run separately). + mapDiffSelected = new Set(outcome.changedTests.filter(t => t in touchfiles)); + removedTests = outcome.removedTests; + } + + // Per-test matching for the remaining changed files + const otherFiles = changedFiles.filter(f => f !== TOUCHFILES_DATA_PATH); const selected: string[] = []; const skipped: string[] = []; for (const [testName, patterns] of Object.entries(touchfiles)) { - const hit = changedFiles.some(f => patterns.some(p => matchGlob(f, p))); + const hit = otherFiles.some(f => patterns.some(p => matchGlob(f, p))) + || (mapDiffSelected !== null && mapDiffSelected.has(testName)); (hit ? selected : skipped).push(testName); } + if (dataChanged) { + return { selected, skipped, reason: 'map-diff', removedTests }; + } return { selected, skipped, reason: 'diff' }; } diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 7ef62b85d..6b211df4a 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -812,9 +812,9 @@ export const GLOBAL_TOUCHFILES = [ 'test/helpers/hermetic-env.ts', // Changes every E2E child's environment 'test/helpers/eval-store.ts', // All E2E tests store results here 'test/helpers/test-selection.ts', // Selection logic itself — a bug here mis-selects every test - // TEMPORARY — maximally conservative until map-diff selection lands: any - // edit to the data maps still forces a full run. A later change-set - // replaces this entry with map-diff (evaluate the old git version of this - // literal-only file, diff the maps, run only the affected tests). - 'test/helpers/touchfiles-data.ts', + // NOTE: this file (touchfiles-data.ts) is deliberately NOT a global + // touchfile. Changes to it route through map-diff selection in + // test-selection.ts: the old git version is evaluated and the maps are + // diffed per key, so a data-only edit runs just the affected tests. + // Map-diff fails CLOSED — any error on that path still runs everything. ]; diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 1ebd336cc..15f369056 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -31,4 +31,13 @@ export { detectBaseBranch, getChangedFiles, selectTests, + diffTouchfileMaps, + diffTouchfileMapsCore, + TOUCHFILES_DATA_PATH, +} from './test-selection'; + +export type { + TouchfileMaps, + MapDiffCause, + MapDiffOutcome, } from './test-selection'; diff --git a/test/touchfiles-map-diff.test.ts b/test/touchfiles-map-diff.test.ts new file mode 100644 index 000000000..896bfe347 --- /dev/null +++ b/test/touchfiles-map-diff.test.ts @@ -0,0 +1,360 @@ +/** + * Map-diff selection for touchfiles-data.ts changes. + * Free (no API calls), runs with `bun test`. + * + * Three layers, matching the injectable-core + thin-shell shape: + * 1. diffTouchfileMapsCore — pure diff logic on injected old/new maps. + * 2. selectTests wiring — injected MapDiffOutcome, no git. + * 3. diffTouchfileMaps shell — real git + bun-child evaluation against a + * throwaway temp repo (happy path + every fail-closed cause), plus one + * end-to-end call against this actual repo's HEAD. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { + diffTouchfileMaps, + diffTouchfileMapsCore, + selectTests, + TOUCHFILES_DATA_PATH, + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, +} from './helpers/touchfiles'; +import type { TouchfileMaps, MapDiffOutcome } from './helpers/touchfiles'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +function maps(overrides: Partial = {}): TouchfileMaps { + return { + E2E_TOUCHFILES: { + 'alpha': ['a/**'], + 'beta': ['b/**', 'shared/util.ts'], + }, + E2E_TIERS: { + 'alpha': 'gate', + 'beta': 'periodic', + }, + LLM_JUDGE_TOUCHFILES: { + 'judge one': ['j/SKILL.md'], + }, + GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'], + ...overrides, + }; +} + +// --- Layer 1: pure core --- + +describe('diffTouchfileMapsCore', () => { + test('identical maps → nothing changed', () => { + const result = diffTouchfileMapsCore(maps(), maps()); + expect(result.changedTests).toEqual([]); + expect(result.removedTests).toEqual([]); + expect(result.globalTouchfilesChanged).toBe(false); + }); + + test('entry added → changed', () => { + const newMaps = maps({ + E2E_TOUCHFILES: { 'alpha': ['a/**'], 'beta': ['b/**', 'shared/util.ts'], 'gamma': ['g/**'] }, + E2E_TIERS: { 'alpha': 'gate', 'beta': 'periodic', 'gamma': 'gate' }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).toEqual(['gamma']); + expect(result.removedTests).toEqual([]); + }); + + test('dep glob edited → changed', () => { + const newMaps = maps({ + E2E_TOUCHFILES: { 'alpha': ['a/**', 'extra/dep.ts'], 'beta': ['b/**', 'shared/util.ts'] }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).toEqual(['alpha']); + }); + + test('tier flipped → changed', () => { + const newMaps = maps({ + E2E_TIERS: { 'alpha': 'gate', 'beta': 'gate' }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).toEqual(['beta']); + }); + + test('unrelated entries untouched → not selected', () => { + const newMaps = maps({ + E2E_TOUCHFILES: { 'alpha': ['a/**', 'x.ts'], 'beta': ['b/**', 'shared/util.ts'] }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).not.toContain('beta'); + expect(result.changedTests).not.toContain('judge one'); + }); + + test('entry removed from every map → removedTests, not changed', () => { + const newMaps = maps({ + E2E_TOUCHFILES: { 'alpha': ['a/**'] }, + E2E_TIERS: { 'alpha': 'gate' }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.removedTests).toEqual(['beta']); + expect(result.changedTests).not.toContain('beta'); + }); + + test('tier entry removed but touchfile entry kept → changed (conservative)', () => { + const newMaps = maps({ + E2E_TIERS: { 'alpha': 'gate' }, // 'beta' tier dropped, E2E_TOUCHFILES.beta kept + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).toContain('beta'); + expect(result.removedTests).toEqual([]); + }); + + test('LLM-judge entries participate in the diff', () => { + const newMaps = maps({ + LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md', 'j/SKILL.md.tmpl'] }, + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.changedTests).toEqual(['judge one']); + }); + + test('GLOBAL_TOUCHFILES entry added → flagged', () => { + const newMaps = maps({ + GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts', 'test/helpers/new-global.ts'], + }); + const result = diffTouchfileMapsCore(maps(), newMaps); + expect(result.globalTouchfilesChanged).toBe(true); + }); + + test('GLOBAL_TOUCHFILES compared as a set — reorder is not a change', () => { + const oldMaps = maps({ GLOBAL_TOUCHFILES: ['x.ts', 'y.ts'] }); + const newMaps = maps({ GLOBAL_TOUCHFILES: ['y.ts', 'x.ts'] }); + const result = diffTouchfileMapsCore(oldMaps, newMaps); + expect(result.globalTouchfilesChanged).toBe(false); + }); +}); + +// --- Layer 2: selectTests wiring (injected outcome, no git) --- + +describe('selectTests map-diff wiring', () => { + const okOutcome = (changedTests: string[], removedTests: string[] = []): MapDiffOutcome => + ({ ok: true, changedTests, removedTests, globalTouchfilesChanged: false }); + + test('data-file change selects only map-changed tests, reason map-diff', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: okOutcome(['browse-basic']) }, + ); + expect(result.selected).toEqual(['browse-basic']); + expect(result.reason).toBe('map-diff'); + expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 1); + }); + + test('map-diff result unions with pattern matching for other changed files', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH, 'retro/SKILL.md'], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: okOutcome(['browse-basic']) }, + ); + expect(result.selected).toContain('browse-basic'); // from map-diff + expect(result.selected).toContain('retro'); // from pattern match + expect(result.selected).toContain('retro-base-branch'); + expect(result.selected).not.toContain('cso-full-audit'); + expect(result.reason).toBe('map-diff'); + }); + + test('changedTests scoped to the map being selected against', () => { + // 'judge one' is an LLM-judge key, not an E2E key — must not leak in. + const result = selectTests( + [TOUCHFILES_DATA_PATH], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: okOutcome(['browse-basic', 'judge one']) }, + ); + expect(result.selected).toEqual(['browse-basic']); + }); + + test('removedTests reported, not selected', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: okOutcome([], ['some-retired-test']) }, + ); + expect(result.selected).toEqual([]); + expect(result.removedTests).toEqual(['some-retired-test']); + }); + + test('FAIL-CLOSED: failed map-diff runs all with cause in reason', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: { ok: false, cause: 'import-failed' } }, + ); + expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length); + expect(result.reason).toBe('global — touchfiles-data changed (import-failed)'); + }); + + test('GLOBAL_TOUCHFILES edit inside data file runs all', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: { ok: true, changedTests: [], removedTests: [], globalTouchfilesChanged: true } }, + ); + expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length); + expect(result.reason).toContain('GLOBAL_TOUCHFILES'); + }); + + test('a real global touchfile hit still wins over map-diff', () => { + const result = selectTests( + [TOUCHFILES_DATA_PATH, 'test/helpers/session-runner.ts'], + E2E_TOUCHFILES, + GLOBAL_TOUCHFILES, + { mapDiff: okOutcome(['browse-basic']) }, + ); + expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length); + expect(result.reason).toBe('global: test/helpers/session-runner.ts'); + }); + + test('no data-file change → classic diff behavior, no map-diff consulted', () => { + const result = selectTests(['retro/SKILL.md'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES, { + // Poison injection: if the wiring consulted this, the test would fail. + mapDiff: { ok: false, cause: 'import-failed' }, + }); + expect(result.reason).toBe('diff'); + expect(result.selected).toContain('retro'); + expect(result.removedTests).toBeUndefined(); + }); +}); + +// --- Layer 3: thin shell against a temp git repo --- + +const OLD_FIXTURE = `export const E2E_TOUCHFILES: Record = { + 'alpha': ['a/**'], + 'beta': ['b/**'], +}; +export const E2E_TIERS: Record = { + 'alpha': 'gate', + 'beta': 'periodic', +}; +export const LLM_JUDGE_TOUCHFILES: Record = { + 'judge one': ['j/SKILL.md'], +}; +export const GLOBAL_TOUCHFILES = [ + 'test/helpers/session-runner.ts', +]; +`; + +describe('diffTouchfileMaps (git + bun-child shell)', () => { + let repo: string; + + const git = (args: string[]) => { + const result = spawnSync( + 'git', + ['-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', '-c', 'tag.gpgsign=false', ...args], + { cwd: repo, stdio: 'pipe', timeout: 10000 }, + ); + if (result.status !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr?.toString()}`); + } + }; + + const commitDataFile = (source: string, message: string) => { + const filePath = path.join(repo, TOUCHFILES_DATA_PATH); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, source); + git(['add', TOUCHFILES_DATA_PATH]); + git(['commit', '-q', '-m', message]); + }; + + beforeAll(() => { + repo = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-repo-')); + git(['init', '-q']); + commitDataFile(OLD_FIXTURE, 'old maps'); + git(['tag', 'old-maps']); + // Commit with a broken data file (unterminated string → bun import fails) + commitDataFile("export const E2E_TOUCHFILES = {\n 'broken: ['\n", 'broken maps'); + git(['tag', 'broken-maps']); + // Commit with wrong shape (tiers value is a number) + commitDataFile( + 'export const E2E_TOUCHFILES = {};\n' + + "export const E2E_TIERS = { 'alpha': 1 };\n" + + 'export const LLM_JUDGE_TOUCHFILES = {};\n' + + 'export const GLOBAL_TOUCHFILES = [];\n', + 'wrong shape', + ); + git(['tag', 'wrong-shape']); + // Commit that deletes the data file entirely (ref exists, file does not) + git(['rm', '-q', TOUCHFILES_DATA_PATH]); + git(['commit', '-q', '-m', 'file deleted']); + git(['tag', 'no-data-file']); + }); + + afterAll(() => { + fs.rmSync(repo, { recursive: true, force: true }); + }); + + test('happy path: old version from git vs injected new maps', () => { + const newMaps: TouchfileMaps = { + E2E_TOUCHFILES: { 'alpha': ['a/**'], 'beta': ['b/**', 'new-dep.ts'], 'gamma': ['g/**'] }, + E2E_TIERS: { 'alpha': 'periodic', 'beta': 'periodic', 'gamma': 'gate' }, + LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md'] }, + GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'], + }; + const outcome = diffTouchfileMaps('old-maps', repo, newMaps); + if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`); + expect(outcome.changedTests).toEqual(['alpha', 'beta', 'gamma']); // tier flip, dep edit, added + expect(outcome.removedTests).toEqual([]); + expect(outcome.globalTouchfilesChanged).toBe(false); + }); + + test('removed key reported from the git version too', () => { + const newMaps: TouchfileMaps = { + E2E_TOUCHFILES: { 'alpha': ['a/**'] }, + E2E_TIERS: { 'alpha': 'gate' }, + LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md'] }, + GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'], + }; + const outcome = diffTouchfileMaps('old-maps', repo, newMaps); + if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`); + expect(outcome.changedTests).toEqual([]); + expect(outcome.removedTests).toEqual(['beta']); + }); + + test('missing base ref → fail-closed with missing-base-ref', () => { + const outcome = diffTouchfileMaps('no-such-ref-anywhere', repo); + expect(outcome).toEqual({ ok: false, cause: 'missing-base-ref' }); + }); + + test('ref exists but file absent → fail-closed with git-show-failed', () => { + const outcome = diffTouchfileMaps('no-data-file', repo); + expect(outcome).toEqual({ ok: false, cause: 'git-show-failed' }); + }); + + test('old file fails to import → fail-closed with import-failed', () => { + const outcome = diffTouchfileMaps('broken-maps', repo); + expect(outcome).toEqual({ ok: false, cause: 'import-failed' }); + }); + + test('old file has unexpected shape → fail-closed with shape-mismatch', () => { + const outcome = diffTouchfileMaps('wrong-shape', repo); + expect(outcome).toEqual({ ok: false, cause: 'shape-mismatch' }); + }); + + test('end-to-end against this repo: HEAD version evaluates and diffs', () => { + // touchfiles-data.ts exists at HEAD (change-set 1). Whatever the working + // tree currently holds, the outcome must be a successful evaluation — + // assert shape, not content, so this stays green before and after the + // change-set commits. + const outcome = diffTouchfileMaps('HEAD', ROOT); + if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`); + expect(Array.isArray(outcome.changedTests)).toBe(true); + expect(Array.isArray(outcome.removedTests)).toBe(true); + expect(typeof outcome.globalTouchfilesChanged).toBe('boolean'); + }); +});