import { describe, expect, test } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; import { CASE_TEST_NAMES, fileCaseRegistration } from '../scripts/lib/paid-cases'; import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, PR_PROFILE_MAPS, caseNameAddressable, formatPrCoverageSummary, packageChangeOnlyVersion, selectPrProfile, validatePrProfileInventory, type PrProfileMaps, } from '../scripts/test-pr-profile'; import { selectTests } from './helpers/touchfiles'; const maps: PrProfileMaps = { e2eTouchfiles: { 'ceo-smoke': ['plan-ceo-review/**'], 'ceo-full': ['plan-ceo-review/**'], 'ceo-periodic': ['plan-ceo-review/**'], 'autoplan-full': ['autoplan/**'], 'browse-smoke': ['browse/**'], }, tiers: { 'ceo-smoke': 'gate', 'ceo-full': 'gate', 'ceo-periodic': 'periodic', 'autoplan-full': 'periodic', 'browse-smoke': 'gate', }, judgeTouchfiles: { 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/**'], 'browse quality': ['browse/**'], }, globalTouchfiles: ['test/helpers/shared-runner.ts'], }; const profile = ['ceo-smoke', 'browse-smoke']; const select = (options: Partial[0]> = {}) => selectPrProfile({ maps, profile, selectedE2E: ['ceo-smoke', 'ceo-full', 'ceo-periodic'], selectedJudges: ['plan-ceo-review/SKILL.md modes'], changedFiles: ['plan-ceo-review/SKILL.md.tmpl'], ...options, }); describe('fast PR coverage policy', () => { test('canonical short probes exist in the unchanged broad gate census', () => { expect(() => validatePrProfileInventory()).not.toThrow(); expect(PR_PROFILE_CASE_IDS).toContain('plan-review-report'); expect(PR_PROFILE_CASE_IDS).toContain('auq-format-gate'); for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic', 'marathon']).toContain(tier); }); test('keeps relevant short probes and the failing CEO quality obligation; reports broad work', () => { const result = select(); expect(result.mode).toBe('pr'); expect(result.e2e).toEqual(['ceo-smoke']); expect(result.judges).toEqual(['plan-ceo-review/SKILL.md modes']); expect(result.deferred.map(({ id, tier }) => ({ id, tier }))).toEqual([ { id: 'ceo-full', tier: 'gate' }, { id: 'ceo-periodic', tier: 'periodic' }, ]); expect(result.needsFullValidation).toBe(false); }); test('the vendored Capy skill (capy/skill/**) is free-only: no paid case reads it, so it needs no full validation', () => { const result = select({ selectedE2E: [], selectedJudges: [], changedFiles: ['capy/skill/gstack/SKILL.md', 'capy/skill/gstack/scripts/install.sh'], derived: null }); expect(result.needsFullValidation).toBe(false); expect(result.missingCoverage).toEqual([]); expect(result.mode).toBe('pr'); expect(result.e2e).toEqual([]); const prompt = select({ selectedE2E: [], selectedJudges: [], changedFiles: ['plan-ceo-review/SKILL.md.tmpl'], derived: null }); expect(prompt.needsFullValidation || prompt.deferredPromptFiles.length > 0 || prompt.e2e.length > 0).toBe(true); }); // Value: protects=a changelog.d fragment (every PR carries one since v1.91.342.0) has no eval dependents, so it never restores the full gate, while a touchfile that names a fragment still selects its case; fails_when=a fragment-only PR selects every gate shard as a "real unknown dependency" (GST-400, #3668); // why_new=knownNonBehaviorFile predates changelog_fragments; seam=.gstack/ship-policy.json changelog_fragments.dir; no_claim=the shadow exercised-path block test('a changelog.d fragment has no eval dependents; a touchfile naming one still selects its case (GST-400)', () => { const fragment = select({ selectedE2E: [], selectedJudges: [], changedFiles: ['changelog.d/gst-400.md', 'changelog.d/README.md'], derived: null }); expect([fragment.mode, fragment.unknownFiles, fragment.e2e, fragment.needsFullValidation]).toEqual(['pr', [], [], false]); const outside = select({ selectedE2E: [], selectedJudges: [], changedFiles: ['changelog.d/x.ts', 'changelog.d/nested/x.md'], derived: null }); expect([...outside.unknownFiles].sort()).toEqual(['changelog.d/nested/x.md', 'changelog.d/x.ts']); const real = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: ['changelog.d/gst-400.md'] }); expect([real.mode, real.unknownFiles]).toEqual(['pr', []]); const table = { ...maps.e2eTouchfiles, 'ceo-smoke': ['plan-ceo-review/**', 'changelog.d/ceo.md'] }; const picked = selectTests(['changelog.d/ceo.md'], table, []).selected; expect(picked).toEqual(['ceo-smoke']); const named = select({ maps: { ...maps, e2eTouchfiles: table }, selectedE2E: picked, selectedJudges: [], changedFiles: ['changelog.d/ceo.md'], derived: null }); expect([named.e2e, named.needsFullValidation]).toEqual([['ceo-smoke'], false]); }); test('does not add unrelated short probes or drop selected quality judges', () => { const result = select({ selectedJudges: ['browse quality', 'plan-ceo-review/SKILL.md modes'] }); expect(result.e2e).not.toContain('browse-smoke'); expect(result.judges).toEqual(['browse quality', 'plan-ceo-review/SKILL.md modes']); }); test('unknown source dependencies restore the full gate and judges, but never launch periodic work', () => { const result = select({ changedFiles: ['lib/new-runtime.ts'] }); expect(result.mode).toBe('full-fallback'); expect(result.unknownFiles).toEqual(['lib/new-runtime.ts']); expect(result.e2e).toEqual(['browse-smoke', 'ceo-full', 'ceo-smoke']); expect(result.judges).toEqual(['browse quality', 'plan-ceo-review/SKILL.md modes']); expect(result.deferred.map(x => x.id)).toEqual(['autoplan-full', 'ceo-periodic']); expect(result.reasons[0]).toContain('Unknown dependencies'); }); test('unknown runner helpers and paid test files also trigger broad fallback', () => { for (const file of ['test/helpers/new-runner.ts', 'test/fixtures/new-prompt.ts', 'test/skill-e2e-new.test.ts', 'bun.lock']) { expect(select({ changedFiles: [file] }).mode).toBe('full-fallback'); } }); test('docs, free tests and touchfile-map edits do not accidentally spend on the full gate', () => { for (const file of ['README.md', 'docs/testing.md', 'VERSION', 'AGENTS.md', 'CLAUDE.md', 'agents-digest/gstack-AGENTS.md', 'test/new-parser.test.ts', 'test/helpers/touchfiles-data.ts']) { expect(select({ changedFiles: [file], selectedE2E: [], selectedJudges: [] }).mode).toBe('pr'); } }); test('only verified package version metadata is safe to ignore', () => { const old = JSON.stringify({ version: '1', scripts: { test: 'bun test' }, dependencies: { bun: '1' } }); expect(packageChangeOnlyVersion(old, old.replace('"version":"1"', '"version":"2"'))).toBe(true); expect(packageChangeOnlyVersion(old, old.replace('bun test', 'bun skip'))).toBe(false); expect(packageChangeOnlyVersion(old, old.replace('"bun":"1"', '"bun":"2"'))).toBe(false); expect(packageChangeOnlyVersion(old, '{}')).toBe(false); expect(packageChangeOnlyVersion(old, 'broken')).toBe(false); }); test('a periodic-only prompt reports deferred coverage instead of a quick-check claim', () => { const result = select({ changedFiles: ['autoplan/SKILL.md.tmpl'], selectedE2E: ['autoplan-full'], selectedJudges: [] }); expect(result.e2e).toEqual([]); expect(result.deferred.map(x => x.id)).toEqual(['autoplan-full']); expect(result.missingCoverage).toEqual([]); expect(result.deferredPromptFiles).toEqual(['autoplan/SKILL.md.tmpl']); expect(result.needsFullValidation).toBe(false); }); test('an unrelated retained check does not satisfy a changed prompt', () => { const result = select({ changedFiles: ['autoplan/sections/phase-close.md'], selectedE2E: ['browse-smoke'], selectedJudges: ['browse quality'] }); expect(result.needsFullValidation).toBe(false); expect(result.missingCoverage).toEqual([]); expect(result.deferredPromptFiles).toEqual(['autoplan/sections/phase-close.md']); }); test('verified template identity covers its generated artifact while reporting the original path', () => { const generatedMaps = { ...maps, e2eTouchfiles: { ...maps.e2eTouchfiles, 'generated-broad': ['generated/SKILL.md.tmpl'] }, tiers: { ...maps.tiers, 'generated-broad': 'periodic' as const }, }; const result = select({ maps: generatedMaps, selectedE2E: ['generated-broad'], selectedJudges: [], changedFiles: ['generated/SKILL.md'], sourceAliases: { 'generated/SKILL.md': 'generated/SKILL.md.tmpl' } }); expect(result.unknownFiles).toEqual([]); expect(result.deferredPromptFiles).toEqual(['generated/SKILL.md']); expect(result.needsFullValidation).toBe(false); expect(select({ maps: generatedMaps, selectedE2E: ['generated-broad'], selectedJudges: [], changedFiles: ['generated/SKILL.md'] }).needsFullValidation).toBe(true); }); test('an unknown new skill is visibly unvalidated even after broad fallback', () => { const result = select({ changedFiles: ['new-skill/SKILL.md.tmpl'] }); expect(result.mode).toBe('full-fallback'); expect(result.needsFullValidation).toBe(true); }); test('a selected judge alone can cover a changed prompt with broad behavioral work deferred', () => { const result = select({ selectedE2E: ['ceo-periodic'], selectedJudges: ['plan-ceo-review/SKILL.md modes'] }); expect(result.e2e).toEqual([]); expect(result.needsFullValidation).toBe(false); expect(result.deferred).toHaveLength(1); }); test('a missing selected prompt dependency is detected even when there are no selected cases', () => { const result = select({ selectedE2E: [], selectedJudges: [] }); expect(result.needsFullValidation).toBe(true); const globalPrompt = select({ maps: { ...maps, globalTouchfiles: ['plan-ceo-review/**'] }, selectedE2E: [], selectedJudges: [] }); expect(globalPrompt.needsFullValidation).toBe(true); }); test('null selections expand inventories while an empty selection remains empty', () => { expect(select({ selectedE2E: null, selectedJudges: null }).e2e).toEqual(['browse-smoke', 'ceo-smoke']); const empty = select({ selectedE2E: [], selectedJudges: [], changedFiles: ['README.md'] }); expect(empty.e2e).toEqual([]); expect(empty.judges).toEqual([]); expect(empty.needsFullValidation).toBe(false); }); test('normalizes Windows paths and deduplicates selection and changes', () => { const result = select({ changedFiles: ['plan-ceo-review\\SKILL.md.tmpl', 'plan-ceo-review/SKILL.md.tmpl'], selectedE2E: ['ceo-smoke', 'ceo-smoke'] }); expect(result.e2e).toEqual(['ceo-smoke']); expect(result.unknownFiles).toEqual([]); expect(result.needsFullValidation).toBe(false); }); test('marathon cases are deferred to their non-blocking lane, even on full fallback', () => { const marathon: PrProfileMaps = { ...maps, tiers: { ...maps.tiers, 'ceo-full': 'marathon' } }; for (const changedFiles of [['plan-ceo-review/SKILL.md.tmpl'], ['lib/new-runtime.ts']]) { const result = select({ maps: marathon, changedFiles }); expect(result.e2e).not.toContain('ceo-full'); expect(result.deferred.find(({ id }) => id === 'ceo-full')).toEqual({ id: 'ceo-full', tier: 'marathon', reason: 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate' }); } expect(() => select({ maps: marathon, profile: ['ceo-full'] })).toThrow('broad gate census'); }); test('every audited PR-profile case is addressable by its Bun test name', () => { // The PR lane runs each audited file with -t built from CASE_TEST_NAMES[id] ?? id; // office-hours-auto-mode's test is named differently and matched 0 tests (#3077 eval-slices run 37741434771). const unaddressable = Object.entries(PR_PROFILE_FILES).flatMap(([file, ids]) => { const source = fs.readFileSync(path.join(import.meta.dir, '..', file), 'utf8'); const computed = fileCaseRegistration(file, source).computed; return ids.filter((id) => { const name = CASE_TEST_NAMES[id] ?? id; return !caseNameAddressable(name, source) && !(computed && source.includes(`'${name}'`)); }).map(id => `${file}: ${id}`); }); expect(unaddressable).toEqual([]); }); test('rejects stale profile IDs, incorrect tiers, missing cadence, and unknown selections', () => { expect(() => select({ profile: ['missing'] })).toThrow('broad gate census'); expect(() => select({ profile: ['ceo-periodic'] })).toThrow('broad gate census'); expect(() => select({ profile: ['ceo-smoke', 'ceo-smoke'] })).toThrow('Duplicate'); expect(() => select({ maps: { ...maps, tiers: { 'ceo-smoke': 'gate' } } })).toThrow('broad gate census'); expect(() => select({ maps: { ...maps, tiers: { ...maps.tiers, 'ceo-full': undefined! } } })).toThrow('no broad'); expect(() => select({ selectedE2E: ['missing'] })).toThrow('Unregistered'); expect(() => select({ selectedJudges: ['missing judge'] })).toThrow('Unregistered'); }); test('job summary states the mode, case counts and reuse; a fallback names each file with its fix (CEO-13)', () => { const fallback = select({ changedFiles: ['lib/new-runtime.ts', 'test/fixtures/new.json'] }); const fallbackLines = formatPrCoverageSummary({ profile: 'pr', selection: { e2e: fallback.e2e, judges: fallback.judges }, prCoverage: fallback }, { total: 9, reused: 4 }).join('\n'); expect(fallbackLines).toContain('- Mode: `full-fallback`'); expect(fallbackLines).toContain('- Selected: 3 E2E case(s), 2 judge(s); 2 deferred'); expect(fallbackLines).toContain('- Reused: 4 of 9 rule/judge record(s)'); expect(fallbackLines).toContain('Full gate restored by 2 file(s) (fix: docs/TESTING_INTERNALS.md#pr-paid-lane-fallback)'); expect(fallbackLines).toContain('`lib/new-runtime.ts` (real unknown dependency): register lib/new-runtime.ts under the cases that consume it'); expect(fallbackLines).toContain('`test/fixtures/new.json` (needs touchfile entry):'); expect(fallbackLines).toContain('FREE_FIXTURES'); const pr = select(); const prLines = formatPrCoverageSummary({ profile: 'pr', selection: { e2e: pr.e2e, judges: pr.judges }, prCoverage: pr }).join('\n'); expect(prLines).toContain('- Mode: `pr`'); expect(prLines).toContain('- Selected: 1 E2E case(s), 1 judge(s); 2 deferred'); expect(prLines).not.toContain('Reused:'); expect(prLines).not.toContain('Full gate restored'); }); test('DX-11: an edited paid test file selects its own name-addressable gate cases outside the profile', () => { const direct: PrProfileMaps = { ...maps, e2eTouchfiles: { ...maps.e2eTouchfiles, 'own-named': ['test/skill-e2e-own.test.ts'], 'own-prose': ['test/skill-e2e-own.test.ts'] }, tiers: { ...maps.tiers, 'own-named': 'gate', 'own-prose': 'gate' } }; const readSource = () => "describeE2E('own', () => {\n testIfSelected('own-named', async () => {});\n test('a prose name', async () => {});\n});"; const result = select({ maps: direct, readSource, changedFiles: ['test/skill-e2e-own.test.ts'], selectedE2E: ['own-named', 'own-prose'], selectedJudges: [] }); expect(result.mode).toBe('pr'); expect(result.e2e).toEqual(['own-named']); expect(result.directCases).toEqual(['own-named']); expect(result.deferred.find(item => item.id === 'own-prose')?.reason).toContain("name its Bun test 'own-prose'"); expect(result.reasons).toContain('Edited paid test files select their own gate cases: own-named'); // A dependency edit (not the case's own test file) still keeps the audited profile only. const viaHelper = select({ maps: { ...direct, e2eTouchfiles: { ...direct.e2eTouchfiles, 'own-named': ['test/skill-e2e-own.test.ts', 'own/**'] } }, readSource, changedFiles: ['own/x.ts'], selectedE2E: ['own-named'], selectedJudges: [] }); expect(viaHelper.e2e).toEqual([]); }); test('DX-11: name addressing ignores comments and near-miss names', () => { expect(caseNameAddressable('a-b', " testIfSelected('a-b', async () => {})")).toBe(true); expect(caseNameAddressable('a-b', ' test(`a-b`, async () => {})')).toBe(true); expect(caseNameAddressable('a-b', ' testConcurrentIfSelected("a-b", async () => {})')).toBe(true); expect(caseNameAddressable('a-b', ' * test (`a-b` in E2E_TIERS)')).toBe(false); expect(caseNameAddressable('a-b', "/* testIfSelected('a-b') */")).toBe(false); expect(caseNameAddressable('a-b', " testIfSelected('a-bc', async () => {})")).toBe(false); }); test('the summary lists derived dependents and files no paid case reaches', () => { const lines = formatPrCoverageSummary({ profile: 'pr', selection: { e2e: ['a'], judges: [] }, prCoverage: { mode: 'dependents', deferred: [], unknownFiles: [], derivedFiles: [{ file: 'design/src/evolve.ts', e2e: 12, judges: 1 }], noConsumerFiles: ['scripts/typecheck-test.ts'] } }).join('\n'); expect(lines).toContain('- Mode: `dependents`'); expect(lines).toContain(' - `design/src/evolve.ts`: 12 case(s), 1 judge(s)'); expect(lines).toContain('- No paid case reaches: `scripts/typecheck-test.ts`'); }); });