From 73fe950fbf439029c518b9df9ce7b03488bce7ae Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 05:31:02 +0000 Subject: [PATCH] feat(evals): parent-computed selection propagates to shard children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sharded runner computed diff selection once, then each of its 48-73 children recomputed it at module load — including, on touchfiles-diff branches, a per-child bun subprocess evaluating the old data file (20s timeout each). The parent now serializes {version, selected, reason} as EVALS_SELECTION_JSON into the shard env; e2e-helpers adopts it at load. Fail-open preserved: any parse/shape violation → ONE stderr warning + local recompute; absent env → silent local compute (non-sharded entrypoints unchanged). Drift test pins parent→child round-trip to identical selection decisions plus the malformed/absent cases. Co-Authored-By: Claude Fable 5 --- test/helpers/e2e-helpers.ts | 50 +++++++++- test/paid-selection-propagation.test.ts | 117 ++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 test/paid-selection-propagation.test.ts diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index ce024a48e..6047dd393 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -61,7 +61,55 @@ export function computeDiffSelection( return selection.selected; } -export let selectedTests: string[] | null = computeDiffSelection(E2E_TOUCHFILES, 'E2E'); // null = run all +/** + * Parse the sharded paid runner's precomputed selection (EVALS_SELECTION_JSON, + * written by serializePaidDiffSelection in scripts/test-paid-shards.ts). + * Returns { selected: null } for run-all. THROWS on any parse/shape failure — + * resolveModuleSelection turns that into a fail-open local recompute. + */ +export function parseEvalsSelectionJson(raw: string): { selected: string[] | null; reason: string } { + const parsed: unknown = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object'); + const { selected, reason } = parsed as { selected?: unknown; reason?: unknown }; + if (selected !== null + && !(Array.isArray(selected) && selected.every((s) => typeof s === 'string'))) { + throw new Error('selected must be null or string[]'); + } + return { + selected: selected as string[] | null, + reason: typeof reason === 'string' ? reason : 'parent selection', + }; +} + +/** + * Resolve the module-load E2E selection: prefer the parent shard runner's + * EVALS_SELECTION_JSON — skipping this module's own git walk and, when + * touchfiles-data.ts is in the diff, the per-child bun subprocess that + * evaluates the old data file (test-selection.ts map-diff path, one per + * shard). On ANY parse/shape failure, fall back to computing locally + * (fail-open preserved) with one stderr warning. + */ +export function resolveModuleSelection( + raw: string | undefined, + compute: () => string[] | null, + stderrWrite: (text: string) => void = (text) => process.stderr.write(text), +): string[] | null { + if (raw) { + try { + const { selected, reason } = parseEvalsSelectionJson(raw); + stderrWrite(`\nE2E selection (parent-propagated: ${reason}): ${selected === null ? 'all' : selected.length} tests\n`); + return selected; + } catch (err) { + stderrWrite(`WARNING: malformed EVALS_SELECTION_JSON (${err instanceof Error ? err.message : String(err)}) — falling back to local selection\n`); + } + } + return compute(); +} + +export let selectedTests: string[] | null = resolveModuleSelection( + evalsEnabled ? process.env.EVALS_SELECTION_JSON : undefined, + () => computeDiffSelection(E2E_TOUCHFILES, 'E2E'), +); // null = run all // EVALS_TIER: filter tests by tier after diff-based selection. // 'gate' = gate tests only (CI default — blocks merge) diff --git a/test/paid-selection-propagation.test.ts b/test/paid-selection-propagation.test.ts new file mode 100644 index 000000000..6666e66a1 --- /dev/null +++ b/test/paid-selection-propagation.test.ts @@ -0,0 +1,117 @@ +/** + * Parent/child selection-drift pins for EVALS_SELECTION_JSON. + * + * The sharded paid runner computes the diff selection ONCE in the parent + * (computePaidDiffSelection in scripts/test-paid-shards.ts), serializes it + * (serializePaidDiffSelection) into every shard child's env, and + * test/helpers/e2e-helpers.ts adopts it at module load (parseEvalsSelectionJson + * via resolveModuleSelection) instead of re-deriving it per shard — which, + * whenever touchfiles-data.ts was in the diff, spawned one bun subprocess PER + * CHILD to evaluate the old data file (test-selection.ts map-diff path). + * + * These pins hold the two sides to IDENTICAL selection decisions across the + * serialize/parse boundary, and the child to fail-open (local recompute with + * one stderr warning) on any parse/shape failure. + */ + +import { describe, test, expect } from 'bun:test'; +import { + computePaidDiffSelection, + serializePaidDiffSelection, + type PaidDiffSelection, +} from '../scripts/test-paid-shards'; +import { parseEvalsSelectionJson, resolveModuleSelection } from './helpers/e2e-helpers'; + +/** The parent's per-test decision shape (PaidDiffSelection.selectedNames). */ +const parentWouldRun = (selection: PaidDiffSelection, name: string): boolean => + selection.selectedNames === null || selection.selectedNames.has(name); + +/** The child's per-test decision shape (testIfSelected / describeIfSelected). */ +const childWouldRun = (selected: string[] | null, name: string): boolean => + selected === null || selected.includes(name); + +const NAMES = ['qa-workflow', 'review-army', 'ship-docsync', 'unmapped-test']; + +describe('EVALS_SELECTION_JSON parent -> child propagation', () => { + test('a concrete selection round-trips to identical decisions', () => { + const fixture: PaidDiffSelection = { + selectedNames: new Set(['qa-workflow', 'ship-docsync']), + reason: 'diff', + totalTests: 4, + }; + const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture)); + expect(parsed.selected).toEqual(['qa-workflow', 'ship-docsync']); + expect(parsed.reason).toBe('diff'); + for (const name of NAMES) { + expect(childWouldRun(parsed.selected, name), name).toBe(parentWouldRun(fixture, name)); + } + }); + + test('run-all (null) round-trips to null — child runs everything', () => { + // computePaidDiffSelection is the REAL parent function; EVALS_ALL is its + // git-free path, so the serializer sees input exactly as produced. + const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv); + expect(selection.selectedNames).toBeNull(); + const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(selection)); + expect(parsed.selected).toBeNull(); + for (const name of NAMES) { + expect(childWouldRun(parsed.selected, name)).toBe(parentWouldRun(selection, name)); + } + }); + + test('empty selection stays empty — nothing selected is NOT run-all', () => { + const fixture: PaidDiffSelection = { selectedNames: new Set(), reason: 'diff', totalTests: 4 }; + const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture)); + expect(parsed.selected).toEqual([]); + for (const name of NAMES) { + expect(childWouldRun(parsed.selected, name)).toBe(false); + expect(parentWouldRun(fixture, name)).toBe(false); + } + }); + + test('parser THROWS on malformed JSON and wrong shapes', () => { + expect(() => parseEvalsSelectionJson('{"selected": ')).toThrow(); + expect(() => parseEvalsSelectionJson('null')).toThrow(); + expect(() => parseEvalsSelectionJson('[1,2]')).toThrow(); + expect(() => parseEvalsSelectionJson('{"selected": 42}')).toThrow(); + expect(() => parseEvalsSelectionJson('{"selected": ["a", 7]}')).toThrow(); + }); + + test('malformed EVALS_SELECTION_JSON falls back to local compute with one stderr warning', () => { + const warnings: string[] = []; + let computed = 0; + const result = resolveModuleSelection( + '{"selected": 42}', + () => { computed += 1; return ['locally-computed']; }, + (text) => warnings.push(text), + ); + expect(result).toEqual(['locally-computed']); // fail-open preserved + expect(computed).toBe(1); + expect(warnings.length).toBe(1); + expect(warnings[0]).toContain('EVALS_SELECTION_JSON'); + }); + + test('absent env var computes locally, silently (non-sharded entrypoints unchanged)', () => { + const writes: string[] = []; + let computed = 0; + const result = resolveModuleSelection( + undefined, + () => { computed += 1; return null; }, + (text) => writes.push(text), + ); + expect(result).toBeNull(); + expect(computed).toBe(1); + expect(writes.length).toBe(0); + }); + + test('a valid env var short-circuits local derivation entirely', () => { + let computed = 0; + const result = resolveModuleSelection( + serializePaidDiffSelection({ selectedNames: new Set(['a']), reason: 'diff', totalTests: 1 }), + () => { computed += 1; return null; }, + () => {}, + ); + expect(result).toEqual(['a']); + expect(computed).toBe(0); // no git walk, no map-diff bun subprocess + }); +});