/** * PR paid-lane classification inventory (ENG-13, CEO-20, W1a). * * Every tracked fixture is either registered by a paid touchfile or declared * free in FREE_FIXTURES, and every workflow is either a paid lane or listed * as free-lane only. A file in neither restores the full gate on every PR * that touches it, so the inventory fails here with the edit that fixes it. */ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { FREE_FIXTURES } from './helpers/free-fixtures-data'; import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data'; import { matchGlob } from './helpers/test-selection'; import { isPaidTestFile } from './helpers/paid-test-set'; import { FREE_ONLY_PR_FILES, PAID_WORKFLOW_FILES, PR_PROFILE_MAPS, selectPrProfile, unknownFileLabel } from '../scripts/test-pr-profile'; import { derivedDependencies } from '../scripts/pr-dependencies'; const ROOT = path.resolve(import.meta.dir, '..'); const tracked = spawnSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8', timeout: 10_000, maxBuffer: 64 * 1024 * 1024 }) .stdout.split('\0').filter(Boolean); const fixtures = tracked.filter(file => file.startsWith('test/fixtures/')); const paidPatterns = [...new Set([...Object.values(E2E_TOUCHFILES).flat(), ...Object.values(LLM_JUDGE_TOUCHFILES).flat(), ...GLOBAL_TOUCHFILES])]; const registered = (file: string) => paidPatterns.some(pattern => matchGlob(file, pattern)); const mapped = (file: string) => Object.keys(FREE_FIXTURES).some(pattern => matchGlob(file, pattern)); const FIX = 'see docs/TESTING_INTERNALS.md#pr-paid-lane-fallback'; describe('FREE_FIXTURES inventory', () => { test('every tracked fixture is registered by a paid touchfile or declared free', () => { const unclassified = fixtures.filter(file => !registered(file) && !mapped(file)); expect(unclassified.map(file => `${file}: neither a paid touchfile nor FREE_FIXTURES lists it. ` + `Fix: register it in the touchfiles of the paid cases that read it (test/helpers/touchfiles-data.ts), ` + `or add '${file}': [''] to FREE_FIXTURES in test/helpers/free-fixtures-data.ts (${FIX})`)).toEqual([]); }); test('every FREE_FIXTURES entry matches a tracked fixture', () => { const stale = Object.keys(FREE_FIXTURES).filter(pattern => !pattern.startsWith('test/fixtures/') || !fixtures.some(file => matchGlob(file, pattern))); expect(stale.map(pattern => `${pattern}: matches no tracked test/fixtures file. Fix: delete the entry from test/helpers/free-fixtures-data.ts (${FIX})`)).toEqual([]); }); test('every declared consumer is free and names its fixture', () => { const problems: string[] = []; for (const [pattern, consumers] of Object.entries(FREE_FIXTURES)) { const name = pattern.endsWith('/**') ? path.basename(pattern.slice(0, -3)) : path.basename(pattern); for (const consumer of consumers) { if (!tracked.includes(consumer)) { problems.push(`${pattern}: consumer ${consumer} is not a tracked file`); continue; } if (isPaidTestFile(consumer) || registered(consumer)) { problems.push(`${pattern}: consumer ${consumer} is paid; move the fixture into that case's touchfiles instead (${FIX})`); } if (!fs.readFileSync(path.join(ROOT, consumer), 'utf8').includes(name)) { problems.push(`${pattern}: consumer ${consumer} never names ${name}; list the file that reads it (${FIX})`); } } } expect(problems).toEqual([]); }); }); describe('workflow inventory (CEO-20)', () => { const workflows = tracked.filter(file => /^\.github\/workflows\/[^/]+\.ya?ml$/.test(file)); const free = (file: string) => FREE_ONLY_PR_FILES.some(pattern => matchGlob(file, pattern)); const paid = (file: string) => (PAID_WORKFLOW_FILES as readonly string[]).includes(file); test('every workflow is classified as paid or free-lane only, never both', () => { const problems = workflows.flatMap(file => { if (paid(file) && free(file)) return [`${file}: listed as both paid and free-lane only; remove it from FREE_ONLY_PR_FILES`]; if (!paid(file) && !free(file)) return [`${file}: unclassified workflow. Fix: add '${file}' to FREE_ONLY_PR_FILES in scripts/test-pr-profile.ts, or to PAID_WORKFLOW_FILES if it runs a paid case (${FIX})`]; return []; }); expect(problems).toEqual([]); }); test('a reusable workflow called by a paid workflow is itself paid', () => { const problems: string[] = []; for (const lane of PAID_WORKFLOW_FILES) { const source = fs.existsSync(path.join(ROOT, lane)) ? fs.readFileSync(path.join(ROOT, lane), 'utf8') : ''; for (const match of source.matchAll(/uses:\s*\.\/(\.github\/workflows\/[^\s@]+\.ya?ml)/g)) { if (!paid(match[1]!)) problems.push(`${lane} calls ${match[1]} via workflow_call; add it to PAID_WORKFLOW_FILES`); } } expect(problems).toEqual([]); expect(workflows.length).toBeGreaterThan(PAID_WORKFLOW_FILES.length); }); test('the inventory rule also reads .yaml files', () => { expect(unknownFileLabel('.github/workflows/new-check.yaml').label).toBe('add to FREE_ONLY_PR_FILES'); expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('full-gate input'); }); }); describe('PR selection over the W1a single-file fallbacks', () => { const select = (changedFiles: string[]) => selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles }); test('scheduler inputs, free-lane workflows and ubicloud scripts stay in the PR profile', () => { // 36497566037 (durations seed), 36485436963 and 36493616568 (free-lane workflow and runner files). for (const changed of [ ['scripts/paid-test-durations.json'], ['scripts/free-test-durations.json'], ['.github/workflows/quality-gate.yml', '.github/workflows/windows-free-tests.yml', 'scripts/free-test-durations.json'], ['scripts/ubicloud/setup-free-suite.sh', 'scripts/ubicloud/ubi-runner.sh'], // 36485436963's replayed diff: a free design test outside test/. ['design/test/variants-retry-after.test.ts', 'make-pdf/test/render.test.ts', 'browse/test/commands.test.ts'], ]) { const result = select(changed); expect({ changed, mode: result.mode, unknown: result.unknownFiles }).toEqual({ changed, mode: 'pr', unknown: [] }); } }); test('seed data and the seed allowlist select no paid case; a derived paid-surface file still selects its dependents (#3289)', () => { // #3289: the allowlist edit derived 246 gate cases, though only test/free-seed-ratchet.test.ts reads it. const seed = select(['scripts/free-test-seed-allowlist.json', 'scripts/free-test-durations.json', 'scripts/free-test-durations-windows.json']); expect({ mode: seed.mode, unknown: seed.unknownFiles, derived: seed.derivedFiles, e2e: seed.e2e, judges: seed.judges }) .toEqual({ mode: 'pr', unknown: [], derived: [], e2e: [], judges: [] }); const paid = select(['design/src/evolve.ts']); expect(paid.derivedFiles.map(entry => entry.file)).toEqual(['design/src/evolve.ts']); expect(paid.e2e.length).toBeGreaterThan(0); }); test('the paid workflow itself still restores the full gate (36493452542)', () => { const result = select(['.github/workflows/evals.yml']); expect(result.mode).toBe('full-fallback'); expect(result.unknownFileLabels).toEqual([{ file: '.github/workflows/evals.yml', ...unknownFileLabel('.github/workflows/evals.yml') }]); expect(result.reasons[0]).toContain('.github/workflows/evals.yml (full-gate input)'); }); test('a mapped free fixture is exempt; a paid-registered fixture selects its cases; an unmapped one falls back', () => { const free = Object.keys(FREE_FIXTURES).find(pattern => !pattern.includes('*') && FREE_FIXTURES[pattern]!.length > 0)!; expect(select([free]).mode).toBe('pr'); const paidFixture = fixtures.find(file => registered(file) && !mapped(file))!; const paidResult = selectPrProfile({ selectedE2E: null, selectedJudges: null, changedFiles: [paidFixture] }); expect(paidResult.unknownFiles).toEqual([]); // A new (tracked) fixture that neither map lists restores the full gate (ENG-13). const real = derivedDependencies(PR_PROFILE_MAPS, ROOT); const unmapped = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: ['test/fixtures/new-unmapped-fixture.json'], derived: { ...real, tracked: new Set([...real.tracked, 'test/fixtures/new-unmapped-fixture.json']) } }); expect(unmapped.mode).toBe('full-fallback'); expect(unmapped.unknownFileLabels[0]).toMatchObject({ label: 'needs touchfile entry' }); expect(unmapped.unknownFileLabels[0]!.fix).toContain('FREE_FIXTURES'); }); test('every fallback label names its fix', () => { for (const file of ['test/helpers/new-helper.ts', 'test/skill-e2e-new.test.ts', 'test/fixtures/x.json', '.github/workflows/new.yml', 'lib/new-runtime.ts']) { const { label, fix } = unknownFileLabel(file); expect(['needs touchfile entry', 'add to FREE_ONLY_PR_FILES', 'real unknown dependency']).toContain(label); expect(fix).toContain(file); expect(fix).toMatch(/touchfiles-data\.ts|FREE_ONLY_PR_FILES|FREE_FIXTURES/); } }); });