/**
* Diff-based test selection for E2E and LLM-judge evals — the LOGIC half.
*
* Each test declares which source files it depends on ("touchfiles") in
* ./touchfiles-data.ts (literals only — see the note there). The test runner
* computes changed files as the union of committed diff, staged + unstaged
* diff, and untracked files — uncommitted work selects tests too — and only
* runs tests whose dependencies were modified. Override with EVALS_ALL=1 to
* run everything.
*
* When touchfiles-data.ts itself changed, selection uses MAP-DIFF instead of
* a global run-all: the old version of the data file is loaded from git and
* evaluated in a bun child process (the literal-only tripwire in
* test/touchfiles-facade.test.ts bounds what executing it can do), the four
* maps are diffed per key, and only tests whose entry was added, whose
* dep-list changed, or whose tier flipped are selected. Any failure on that
* path fails CLOSED: run all tests, with the cause in the reason string.
*
* Everything here is synchronous by design: e2e-helpers.ts and the *-e2e
* test files compute selection at module load, so the old-file evaluation
* happens in a spawnSync'd bun child rather than a dynamic import.
*
* Import sites should keep using the ./touchfiles facade, which re-exports
* both this module and the data module.
*/
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import {
E2E_TOUCHFILES,
E2E_TIERS,
LLM_JUDGE_TOUCHFILES,
GLOBAL_TOUCHFILES,
} from './touchfiles-data';
/** Repo-relative path of the pure-data file (the map-diff subject). */
export const TOUCHFILES_DATA_PATH = 'test/helpers/touchfiles-data.ts';
// --- Glob matching ---
/**
* Match a file path against a glob pattern.
* Supports:
* ** — match any number of path segments
* * — match within a single segment (no /)
*/
export function matchGlob(file: string, pattern: string): boolean {
const regexStr = pattern
.replace(/\./g, '\\.')
.replace(/\*\*/g, '{{GLOBSTAR}}')
.replace(/\*/g, '[^/]*')
.replace(/\{\{GLOBSTAR\}\}/g, '.*');
return new RegExp(`^${regexStr}$`).test(file);
}
// --- Base branch detection ---
/**
* Detect the base branch by trying refs in order.
* Returns the first valid ref, or null if none found.
*/
export function detectBaseBranch(cwd: string): string | null {
for (const ref of ['origin/main', 'origin/master', 'main', 'master']) {
const result = spawnSync('git', ['rev-parse', '--verify', ref], {
cwd, stdio: 'pipe', timeout: 3000,
});
if (result.status === 0) return ref;
}
return null;
}
/**
* Run a git command and return stdout. FAIL-CLOSED: any failure (spawn
* error, non-zero exit) throws — a broken git environment must abort the
* suite loudly instead of silently degrading into a full (paid) run.
*/
function runGitOrThrow(args: string[], cwd: string, spawnImpl: typeof spawnSync): string {
const result = spawnImpl('git', args, {
cwd, stdio: 'pipe', timeout: 10000, maxBuffer: 8 * 1024 * 1024,
});
if (result.error || result.status !== 0) {
const stderr = result.stderr?.toString().trim() || result.error?.message || 'unknown error';
throw new Error(
`getChangedFiles: \`git ${args.join(' ')}\` failed in ${cwd} `
+ `(exit ${result.status ?? 'spawn-error'}): ${stderr}\n`
+ 'Diff-based test selection cannot proceed. Fix the git environment, '
+ 'or set EVALS_ALL=1 to deliberately run the full suite.',
);
}
return result.stdout.toString();
}
/**
* Get the list of files changed relative to the base branch, INCLUDING
* uncommitted work. Union of three sources, deduped:
* 1. committed: `git diff --name-only ...HEAD`
* 2. staged + unstaged: `git diff --name-only HEAD`
* 3. untracked: `git status --porcelain --untracked-files=all` ('?? ' lines)
*
* Without 2 and 3, an agent that edits files and runs evals BEFORE
* committing gets an empty diff → run-all → the full paid suite every time.
*
* An empty UNION still means "no changes" and callers keep their intentional
* run-all semantics for it (main-branch / periodic full runs depend on that).
*
* Git failures THROW (see runGitOrThrow) instead of returning [] — the old
* behavior made a broken git environment indistinguishable from a clean tree.
*
* `spawnImpl` is injectable for tests.
*/
export function getChangedFiles(
baseBranch: string,
cwd: string,
spawnImpl: typeof spawnSync = spawnSync,
): string[] {
// core.quotePath=false: without it git C-escapes non-ASCII bytes
// ("docs/r\303\251sum\303\251.md"), the escaped string matches no glob,
// and the dependent test is silently DESELECTED — under-selection, the
// exact direction selection must fail away from.
const noQuote = ['-c', 'core.quotePath=false'];
const committed = runGitOrThrow([...noQuote, 'diff', '--name-only', `${baseBranch}...HEAD`], cwd, spawnImpl)
.trim().split('\n').filter(Boolean);
const uncommitted = runGitOrThrow([...noQuote, 'diff', '--name-only', 'HEAD'], cwd, spawnImpl)
.trim().split('\n').filter(Boolean);
const untracked = runGitOrThrow([...noQuote, 'status', '--porcelain', '--untracked-files=all'], cwd, spawnImpl)
.split('\n')
.filter(line => line.startsWith('?? '))
.map(line => {
let p = line.slice(3);
// residual quoting (embedded quote/newline) — strip the wrapper
if (p.startsWith('"') && p.endsWith('"')) p = p.slice(1, -1);
return p;
});
return [...new Set([...committed, ...uncommitted, ...untracked])];
}
// --- Touchfile map diffing ---
/** The four exports of touchfiles-data.ts, as plain data. */
export interface TouchfileMaps {
E2E_TOUCHFILES: Record;
E2E_TIERS: Record;
LLM_JUDGE_TOUCHFILES: Record;
GLOBAL_TOUCHFILES: string[];
}
export type MapDiffCause =
| 'missing-base-ref'
| 'git-show-failed'
| 'import-failed'
| 'shape-mismatch';
export type MapDiffOutcome =
| {
ok: true;
/** Tests whose entry was added, dep-list changed, or tier flipped. */
changedTests: string[];
/** Keys present in the old maps but gone from every new map (reported, not selected). */
removedTests: string[];
/** True when the GLOBAL_TOUCHFILES set itself changed — not attributable to any test. */
globalTouchfilesChanged: boolean;
}
| { ok: false; cause: MapDiffCause };
/** Current maps as a TouchfileMaps value (the "new" side of the diff). */
const CURRENT_MAPS: TouchfileMaps = {
E2E_TOUCHFILES,
E2E_TIERS,
LLM_JUDGE_TOUCHFILES,
GLOBAL_TOUCHFILES,
};
function isStringArray(v: unknown): v is string[] {
return Array.isArray(v) && v.every(x => typeof x === 'string');
}
function isRecordOfStringArrays(v: unknown): v is Record {
return !!v && typeof v === 'object' && !Array.isArray(v)
&& Object.values(v).every(isStringArray);
}
function isRecordOfStrings(v: unknown): v is Record {
return !!v && typeof v === 'object' && !Array.isArray(v)
&& Object.values(v).every(x => typeof x === 'string');
}
function isTouchfileMaps(v: unknown): v is TouchfileMaps {
if (!v || typeof v !== 'object') return false;
const o = v as Record;
return isRecordOfStringArrays(o.E2E_TOUCHFILES)
&& isRecordOfStrings(o.E2E_TIERS)
&& isRecordOfStringArrays(o.LLM_JUDGE_TOUCHFILES)
&& isStringArray(o.GLOBAL_TOUCHFILES);
}
/**
* Pure map-diff core (injectable for tests — no git, no filesystem).
*
* A key counts as CHANGED when it was added to any per-key map, its dep-list
* array differs, or its tier value flipped. A key counts as REMOVED only when
* it is gone from every new per-key map; a key dropped from one map but still
* present in another (e.g. tier entry deleted, touchfile entry kept) counts
* as changed — conservative, because the test still exists with a different
* configuration. GLOBAL_TOUCHFILES is compared as a set; a change there is
* not attributable to any test and is flagged for the caller to treat as
* "run all".
*/
export function diffTouchfileMapsCore(
oldMaps: TouchfileMaps,
newMaps: TouchfileMaps,
): { changedTests: string[]; removedTests: string[]; globalTouchfilesChanged: boolean } {
const perKeyMapNames = ['E2E_TOUCHFILES', 'E2E_TIERS', 'LLM_JUDGE_TOUCHFILES'] as const;
const changed = new Set();
const rawRemoved = new Set();
for (const mapName of perKeyMapNames) {
const oldMap: Record = oldMaps[mapName] ?? {};
const newMap: Record = newMaps[mapName] ?? {};
for (const key of Object.keys(newMap)) {
if (!(key in oldMap)) {
changed.add(key); // added
} else if (JSON.stringify(oldMap[key]) !== JSON.stringify(newMap[key])) {
changed.add(key); // dep-list edited or tier flipped
}
}
for (const key of Object.keys(oldMap)) {
if (!(key in newMap)) rawRemoved.add(key);
}
}
const removed = new Set();
for (const key of rawRemoved) {
const stillExists = perKeyMapNames.some(m => key in (newMaps[m] ?? {}));
if (stillExists) changed.add(key);
else removed.add(key);
}
const sortedSet = (arr: string[]) => JSON.stringify([...arr].sort());
const globalTouchfilesChanged =
sortedSet(oldMaps.GLOBAL_TOUCHFILES ?? []) !== sortedSet(newMaps.GLOBAL_TOUCHFILES ?? []);
return {
changedTests: [...changed].sort(),
removedTests: [...removed].sort(),
globalTouchfilesChanged,
};
}
/**
* Load the OLD touchfiles-data.ts from git and diff it against the current
* maps. Synchronous: the old file is written to a temp dir and evaluated in
* a spawnSync'd bun child that prints the four maps as JSON (module-scope
* callers like e2e-helpers.ts cannot await).
*
* FAIL-CLOSED: every failure returns `{ ok: false, cause }` and the caller
* must treat that as "data change is global — run all tests".
*
* `newMaps` is injectable so integration tests can diff a temp repo's old
* version against a fixture instead of this repo's live maps.
*/
export function diffTouchfileMaps(
baseRef: string,
cwd: string,
newMaps: TouchfileMaps = CURRENT_MAPS,
): MapDiffOutcome {
try {
const verify = spawnSync('git', ['rev-parse', '--verify', baseRef], {
cwd, stdio: 'pipe', timeout: 3000,
});
if (verify.status !== 0) return { ok: false, cause: 'missing-base-ref' };
const show = spawnSync('git', ['show', `${baseRef}:${TOUCHFILES_DATA_PATH}`], {
cwd, stdio: 'pipe', timeout: 5000, maxBuffer: 8 * 1024 * 1024,
});
if (show.status !== 0) return { ok: false, cause: 'git-show-failed' };
const oldSource = show.stdout.toString();
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-'));
try {
const dataPath = path.join(tempDir, 'touchfiles-data.ts');
fs.writeFileSync(dataPath, oldSource);
const loaderPath = path.join(tempDir, 'load-maps.ts');
fs.writeFileSync(loaderPath, [
`const m = await import(${JSON.stringify(dataPath)});`,
'console.log(JSON.stringify({',
' E2E_TOUCHFILES: m.E2E_TOUCHFILES,',
' E2E_TIERS: m.E2E_TIERS,',
' LLM_JUDGE_TOUCHFILES: m.LLM_JUDGE_TOUCHFILES,',
' GLOBAL_TOUCHFILES: m.GLOBAL_TOUCHFILES,',
'}));',
'',
].join('\n'));
// process.execPath is the bun binary when running under bun.
const run = spawnSync(process.execPath, ['run', loaderPath], {
stdio: 'pipe', timeout: 20000, maxBuffer: 8 * 1024 * 1024,
});
if (run.status !== 0) return { ok: false, cause: 'import-failed' };
let oldMaps: unknown;
try {
oldMaps = JSON.parse(run.stdout.toString());
} catch {
return { ok: false, cause: 'import-failed' };
}
if (!isTouchfileMaps(oldMaps)) return { ok: false, cause: 'shape-mismatch' };
return { ok: true, ...diffTouchfileMapsCore(oldMaps, newMaps) };
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
} catch {
// Unexpected failure anywhere in the pipeline (temp dir, git, child
// process) — same fail-closed contract as an evaluation failure.
return { ok: false, cause: 'import-failed' };
}
}
// --- Test selection ---
/**
* Select tests to run based on changed files.
*
* Algorithm:
* 1. If any changed file (other than touchfiles-data.ts) matches a global
* touchfile → run ALL tests
* 2. If touchfiles-data.ts changed → map-diff it against the base ref and
* select only the tests whose map entries changed (fail-closed: any
* map-diff failure runs ALL tests, with the cause in the reason string)
* 3. For each test, check if any other changed file matches its patterns
* 4. Return selected + skipped lists with reason (union of 2 and 3)
*
* `opts.baseRef` / `opts.cwd` scope the map-diff; they default to
* EVALS_BASE || detectBaseBranch || 'main' and the repo root — the same
* resolution the module-scope callers (e2e-helpers.ts et al.) use to compute
* `changedFiles`, so the two sides of the diff stay consistent.
* `opts.mapDiff` injects a precomputed outcome (for tests).
*/
export function selectTests(
changedFiles: string[],
touchfiles: Record,
globalTouchfiles: string[] = GLOBAL_TOUCHFILES,
opts: { baseRef?: string; cwd?: string; mapDiff?: MapDiffOutcome } = {},
): { selected: string[]; skipped: string[]; reason: string; removedTests?: string[] } {
const allTestNames = Object.keys(touchfiles);
const dataChanged = changedFiles.includes(TOUCHFILES_DATA_PATH);
// Global touchfile hit → run all. touchfiles-data.ts is excluded here —
// its changes route through map-diff below instead of a global run-all.
for (const file of changedFiles) {
if (file === TOUCHFILES_DATA_PATH) continue;
if (globalTouchfiles.some(g => matchGlob(file, g))) {
return { selected: allTestNames, skipped: [], reason: `global: ${file}` };
}
}
// Map-diff path for data-file changes
let mapDiffSelected: Set | null = null;
let removedTests: string[] | undefined;
if (dataChanged) {
const cwd = opts.cwd ?? path.resolve(import.meta.dir, '..', '..');
const baseRef = opts.baseRef
|| process.env.EVALS_BASE
|| detectBaseBranch(cwd)
|| 'main';
const outcome = opts.mapDiff ?? diffTouchfileMaps(baseRef, cwd);
if (!outcome.ok) {
return {
selected: allTestNames,
skipped: [],
reason: `global — touchfiles-data changed (${outcome.cause})`,
};
}
if (outcome.globalTouchfilesChanged) {
return {
selected: allTestNames,
skipped: [],
reason: 'global — touchfiles-data changed (GLOBAL_TOUCHFILES edited)',
};
}
// Scope to this map's keys (E2E and LLM-judge selections run separately).
mapDiffSelected = new Set(outcome.changedTests.filter(t => t in touchfiles));
removedTests = outcome.removedTests;
}
// Per-test matching for the remaining changed files
const otherFiles = changedFiles.filter(f => f !== TOUCHFILES_DATA_PATH);
const selected: string[] = [];
const skipped: string[] = [];
for (const [testName, patterns] of Object.entries(touchfiles)) {
const hit = otherFiles.some(f => patterns.some(p => matchGlob(f, p)))
|| (mapDiffSelected !== null && mapDiffSelected.has(testName));
(hit ? selected : skipped).push(testName);
}
if (dataChanged) {
return { selected, skipped, reason: 'map-diff', removedTests };
}
return { selected, skipped, reason: 'diff' };
}