diff --git a/test/fixtures/moved-module-selection/w3-browse-routes.json b/test/fixtures/moved-module-selection/w3-browse-routes.json new file mode 100644 index 000000000..2ff76d858 --- /dev/null +++ b/test/fixtures/moved-module-selection/w3-browse-routes.json @@ -0,0 +1,40 @@ +{ + "workstream": "W3 browse route table", + "recordedAt": "96764e80", + "source": "browse/src/server.ts", + "modules": [ + "browse/src/routes/activity.ts", + "browse/src/routes/commands.ts", + "browse/src/routes/core.ts", + "browse/src/routes/files.ts", + "browse/src/routes/index.ts", + "browse/src/routes/inspector.ts", + "browse/src/routes/pairing.ts", + "browse/src/routes/pty.ts", + "browse/src/routes/table.ts", + "browse/src/routes/tokens.ts", + "browse/src/routes/tunnel.ts" + ], + "e2e": [ + "aside-browse-basic", + "aside-browse-flow", + "benchmark-workflow", + "browse-basic", + "browse-snapshot", + "canary-workflow", + "carve-section-loading", + "design-review-detector-shim-dom", + "design-review-fix", + "diagram-authoring-quality", + "diagram-triplet", + "qa-b6-static", + "qa-b7-spa", + "qa-b8-checkout", + "qa-fix-loop", + "qa-only-no-fix", + "qa-quick" + ], + "llmJudge": [ + "browse/SKILL.md reference" + ] +} diff --git a/test/touchfile-moved-module-selection.test.ts b/test/touchfile-moved-module-selection.test.ts new file mode 100644 index 000000000..1b7421e38 --- /dev/null +++ b/test/touchfile-moved-module-selection.test.ts @@ -0,0 +1,61 @@ +/** + * Touchfile coverage for moved code. + * + * Each JSON file in test/fixtures/moved-module-selection/ records, before a + * refactor moved code out of `source`, the paid evals selected by touching + * that source file. Touching any module the code moved into must select a + * superset, so a move can never silently drop eval coverage. + */ + +import { describe, test, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const GOLDEN_DIR = 'test/fixtures/moved-module-selection'; + +interface Golden { workstream: string; source: string; modules: string[]; e2e: string[]; llmJudge: string[] } + +const goldens = fs.readdirSync(path.join(ROOT, GOLDEN_DIR)) + .filter(f => f.endsWith('.json')) + .map(f => ({ file: `${GOLDEN_DIR}/${f}`, golden: JSON.parse(fs.readFileSync(path.join(ROOT, GOLDEN_DIR, f), 'utf-8')) as Golden })); + +function missingSelections(module: string, golden: Golden): string[] { + const e2e = new Set(selectTests([module], E2E_TOUCHFILES, GLOBAL_TOUCHFILES).selected); + const judge = new Set(selectTests([module], LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES).selected); + return [ + ...golden.e2e.filter(t => !e2e.has(t)).map(t => `E2E ${t}`), + ...golden.llmJudge.filter(t => !judge.has(t)).map(t => `LLM judge ${t}`), + ]; +} + +describe('moved modules keep their source file\'s paid-eval selection', () => { + test('at least one golden is recorded', () => { + expect(goldens.length).toBeGreaterThan(0); + }); + + for (const { file, golden } of goldens) { + test(`${golden.workstream}: every module moved out of ${golden.source} selects a superset`, () => { + const problems: string[] = []; + for (const module of golden.modules) { + if (!fs.existsSync(path.join(ROOT, module))) problems.push(`${module} (listed in ${file} but missing)`); + for (const miss of missingSelections(module, golden)) problems.push(`${module} does not select ${miss}`); + } + if (problems.length) { + throw new Error([ + 'Moved modules lost paid-eval selection:', + ...problems.map(p => ` ${p}`), + `Rule: code moved out of ${golden.source} must keep every eval that touching ${golden.source} selected, or a regression in the moved code ships unevaluated.`, + `Fix: add the new module to ${golden.source}'s touchfile entries in test/helpers/touchfiles-data.ts (a directory glob is fine).`, + `Golden: ${file} — re-record it only when the source file's own selection legitimately changes.`, + ].join('\n')); + } + }); + } + + test('a module outside every touchfile entry is reported', () => { + const golden: Golden = { workstream: 'self-test', source: 'browse/src/server.ts', modules: [], e2e: ['browse-basic'], llmJudge: [] }; + expect(missingSelections('nowhere/planted-module.ts', golden)).toEqual(['E2E browse-basic']); + }); +});