From 011d02ac979c1520f0f8ee750a4d41dd4b10ad4c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:55:48 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20census=20integrity=20=E2=80=94=2017=20ph?= =?UTF-8?q?antom=20selection=20keys=20deleted,=20reverse=20invariant=20add?= =?UTF-8?q?ed,=20gitignored=20dep=20patterns=20replaced,=20local=20map=20f?= =?UTF-8?q?orks=20derived?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge-blocking gate census counted tests that could not run. Deleted (critic-verified against both quoted-occurrence and dep-registration liveness): 7 *-prosons-format keys with no declaring test, ship-plan- completion/-verification, review-plan-completion, design-shotgun-path/ session/full, autoplan-core (dead ~10 months), e2e-harness-audit (its namesake is a FREE-suite file), plus 2 dead LLM-judge keys and 2 free-file keys (budget-regression-pty, global-discover) misplaced in the PAID maps. Census: 191 -> 174 keys, gate 86 -> 78 honest. The new reverse invariant in touchfiles.test.ts makes the class structurally impossible: every key must be quoted in a living paid test file OR registered to an existing paid test file via its dep list (the constructed- name binding the 2026-08 self-registration sweep established) — zero exceptions needed today, with a live-file check on any future exception. Also: '.agents/skills/**' dep patterns replaced with the generator (scripts/gen-skill-docs.ts) — .agents/ is gitignored, so those patterns could NEVER match a git diff and review-template edits silently stopped selecting codex/gemini tests; the codex/gemini local touchfile maps are now DERIVED from the canonical map (loud throw if a key vanishes) instead of hand-forked copies that had already drifted. ios-qa-e2e demoted gate -> periodic: its gate declaration was never executable in CI (hardware exclusion only applies at tier=periodic), so every Linux PR planned a hollow shard. Co-Authored-By: Claude Fable 5 --- test/codex-e2e.test.ts | 17 ++++++--- test/gemini-e2e.test.ts | 16 +++++--- test/helpers/touchfiles-data.ts | 53 ++++++------------------- test/touchfiles.test.ts | 68 +++++++++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 56 deletions(-) diff --git a/test/codex-e2e.test.ts b/test/codex-e2e.test.ts index 2b8fc55fc..1e4b0eeb3 100644 --- a/test/codex-e2e.test.ts +++ b/test/codex-e2e.test.ts @@ -63,11 +63,18 @@ if (!evalsEnabled) { // --- Diff-based test selection --- -// Codex E2E touchfiles — keyed by test name, same pattern as E2E_TOUCHFILES -const CODEX_E2E_TOUCHFILES: Record = { - 'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts'], - 'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts'], -}; +// Codex E2E touchfiles — DERIVED from the canonical map, never a local fork. +// The old hand-copy drifted (it kept gitignored '.agents/skills/**' patterns +// that can never match a git diff, and missed deps the canonical map gained +// like lib/worktree.ts and this test file itself), so review-template edits +// silently stopped selecting these tests. Deriving keeps one source of truth +// and puts these keys under the tier-alignment + dep-existence invariants. +const CODEX_E2E_TOUCHFILES: Record = Object.fromEntries( + (['codex-discover-skill', 'codex-review-findings'] as const).map((key) => { + if (!E2E_TOUCHFILES[key]) throw new Error(`canonical E2E_TOUCHFILES lost key '${key}' — fix the map, not this file`); + return [key, E2E_TOUCHFILES[key]]; + }), +); let selectedTests: string[] | null = null; // null = run all diff --git a/test/gemini-e2e.test.ts b/test/gemini-e2e.test.ts index cdd9a9fa5..ed1270a61 100644 --- a/test/gemini-e2e.test.ts +++ b/test/gemini-e2e.test.ts @@ -19,7 +19,7 @@ import { JUDGE_MS } from './helpers/eval-budgets'; import { runGeminiSkill } from './helpers/gemini-session-runner'; import type { GeminiResult } from './helpers/gemini-session-runner'; import { EvalCollector } from './helpers/eval-store'; -import { selectTests, detectBaseBranch, getChangedFiles, GLOBAL_TOUCHFILES } from './helpers/touchfiles'; +import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles'; import { createTestWorktree, harvestAndCleanup } from './helpers/e2e-helpers'; import * as path from 'path'; @@ -74,10 +74,16 @@ if (!evalsEnabled) { // --- Diff-based test selection --- -// Gemini E2E touchfiles — keyed by test name -const GEMINI_E2E_TOUCHFILES: Record = { - 'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts'], -}; +// Gemini E2E touchfiles — DERIVED from the canonical map, never a local fork +// (the old hand-copy kept a gitignored '.agents/skills/**' pattern that can +// never match a git diff and missed canonical deps — same drift class as the +// codex copy). +const GEMINI_E2E_TOUCHFILES: Record = Object.fromEntries( + (['gemini-smoke'] as const).map((key) => { + if (!E2E_TOUCHFILES[key]) throw new Error(`canonical E2E_TOUCHFILES lost key '${key}' — fix the map, not this file`); + return [key, E2E_TOUCHFILES[key]]; + }), +); let selectedTests: string[] | null = null; // null = run all diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index b3f714f19..04c8e81f9 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -134,7 +134,6 @@ export const E2E_TOUCHFILES: Record = { 'auq-repetition-cut-ab': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/llm-judge.ts', 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md', 'test/skill-e2e-auq-repetition-cut-ab.test.ts'], 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], - 'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'], 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'], 'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], @@ -149,7 +148,6 @@ export const E2E_TOUCHFILES: Record = { // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'], 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], - 'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + @@ -207,13 +205,6 @@ export const E2E_TOUCHFILES: Record = { 'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'], // Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble - 'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'office-hours-prosons-format': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'investigate-prosons-format': ['investigate/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'qa-prosons-format': ['qa/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'review-prosons-format': ['review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'design-review-prosons-format': ['design-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], // /plan-tune (v1 observational) 'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile', 'test/skill-e2e-plan-tune.test.ts'], @@ -241,7 +232,6 @@ export const E2E_TOUCHFILES: Record = { 'retro-base-branch': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'], // Global discover - 'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'], // CSO 'cso-full-audit': ['cso/**', 'test/skill-e2e-cso.test.ts'], @@ -277,14 +267,14 @@ export const E2E_TOUCHFILES: Record = { 'codex-review': ['codex/**', 'test/skill-e2e-workflow.test.ts'], // Codex E2E (tests skills via Codex CLI + worktree) - 'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], - 'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], + 'codex-discover-skill': ['codex/**', 'scripts/gen-skill-docs.ts', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], + 'codex-review-findings': ['review/**', 'scripts/gen-skill-docs.ts', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], // GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill) 'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'], // Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks) - 'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'], + 'gemini-smoke': ['scripts/gen-skill-docs.ts', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'], // Coverage audit (shared fixture) + triage + gates @@ -295,9 +285,6 @@ export const E2E_TOUCHFILES: Record = { 'ship-docsync': ['ship/**', 'document-release/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/sections.ts', 'test/skill-e2e-ship-docsync.test.ts'], // Plan completion audit + verification - 'ship-plan-completion': ['ship/**', 'scripts/gen-skill-docs.ts'], - 'ship-plan-verification': ['ship/**', 'qa-only/**', 'scripts/gen-skill-docs.ts'], - 'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'], // Design 'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'], @@ -308,9 +295,6 @@ export const E2E_TOUCHFILES: Record = { 'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], // Design Shotgun - 'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'], - 'design-shotgun-session': ['design-shotgun/**', 'scripts/resolvers/design.ts'], - 'design-shotgun-full': ['design-shotgun/**', 'design/src/**', 'browse/src/**'], // /diagram (diagram-render bundle consumers). Triplet = deterministic // functional (gate); authoring quality = LLM-judged benchmark (periodic). @@ -330,7 +314,6 @@ export const E2E_TOUCHFILES: Record = { // Autoplan - 'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'], 'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts'], // Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs @@ -523,7 +506,6 @@ export const E2E_TIERS: Record = { 'review-base-branch': 'gate', 'review-design-lite': 'periodic', // 4/7 threshold is subjective 'review-coverage-audit': 'gate', - 'review-plan-completion': 'gate', 'review-dashboard-via': 'gate', // Review Army — gate for core functionality, periodic for multi-specialist @@ -583,7 +565,6 @@ export const E2E_TIERS: Record = { 'office-hours-auto-mode': 'gate', 'auto-decide-preserved': 'periodic', 'conductor-prose': 'periodic', - 'e2e-harness-audit': 'gate', // Real-PTY E2E batch — tier classification: // gate: cheap, deterministic, run on every PR @@ -593,7 +574,6 @@ export const E2E_TIERS: Record = { 'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe 'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions 'plan-design-with-ui-scope': 'gate', // ~$0.80/run - 'budget-regression-pty': 'gate', // free, library-only assertion 'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode 'tpa-present': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts 'tpa-absent-linux': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts @@ -656,13 +636,6 @@ export const E2E_TIERS: Record = { 'plan-review-prosons-neutral-neg': 'periodic', // CT3 expanded coverage — non-plan-review skills inheriting Pros/Cons (all periodic) - 'ship-prosons-format': 'periodic', - 'office-hours-prosons-format': 'periodic', - 'investigate-prosons-format': 'periodic', - 'qa-prosons-format': 'periodic', - 'review-prosons-format': 'periodic', - 'design-review-prosons-format': 'periodic', - 'document-release-prosons-format': 'periodic', // /plan-tune — gate (core v1 DX promise: plain-English intent routing) 'plan-tune-inspect': 'gate', @@ -702,15 +675,12 @@ export const E2E_TIERS: Record = { 'ship-coverage-audit': 'gate', 'ship-triage': 'gate', 'ship-docsync': 'gate', - 'ship-plan-completion': 'gate', - 'ship-plan-verification': 'gate', // Retro — gate for cheap branch detection, periodic for full Opus retro 'retro': 'periodic', 'retro-base-branch': 'gate', // Global discover - 'global-discover': 'gate', // CSO — gate for security guardrails, periodic for quality 'cso-full-audit': 'periodic', // D2a demotion 2026-08: 250s/$0.57 full audit; cso targeted tests stay gate @@ -739,9 +709,6 @@ export const E2E_TIERS: Record = { 'design-consultation-preview': 'periodic', // D2a demotion 2026-08 ($0.89/481s) 'plan-design-review-no-ui-scope': 'gate', 'design-review-fix': 'periodic', - 'design-shotgun-path': 'gate', - 'design-shotgun-session': 'gate', - 'design-shotgun-full': 'periodic', // /diagram — triplet is deterministic functional, judge is a quality benchmark 'diagram-triplet': 'gate', @@ -760,7 +727,6 @@ export const E2E_TIERS: Record = { // Autoplan — periodic (not yet implemented) - 'autoplan-core': 'periodic', 'autoplan-dual-voice': 'periodic', // Multi-provider benchmark — periodic (requires external CLIs + auth, paid) @@ -793,9 +759,14 @@ export const E2E_TIERS: Record = { 'overlay-harness-opus-4-7-fanout-toy': 'periodic', 'overlay-harness-opus-4-7-fanout-realistic': 'periodic', - // /ios-qa daemon + codegen — no-device path runs every PR (no hardware - // dependency, deterministic). with-device path requires GSTACK_HAS_IOS_DEVICE. - 'ios-qa-e2e': 'gate', + // /ios-qa daemon + codegen. Demoted gate -> periodic (2026-08 audit): the + // gate declaration was never executable in CI — the file sits in + // PERIODIC_CI_EXCLUDE ("not a CI runner capability"), but that exclusion + // only applies at tier=periodic, so the gate lane planned a HOLLOW shard + // on every Linux PR. Periodic keeps it in the weekly census on capable + // hosts; re-promote if a macOS runner lands (flagged decision in the + // test-infra overhaul plan). + 'ios-qa-e2e': 'periodic', // Swift toolchain only, no device required, but heavier than TS unit tests. 'ios-qa-swift-build': 'periodic', // Requires a real connected + paired iPhone. Manual-trigger only. @@ -841,8 +812,6 @@ export const LLM_JUDGE_TOUCHFILES: Record = { 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Office Hours - 'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], // Deploy skills 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'], diff --git a/test/touchfiles.test.ts b/test/touchfiles.test.ts index 6bf5fb25c..ac9b652ca 100644 --- a/test/touchfiles.test.ts +++ b/test/touchfiles.test.ts @@ -81,14 +81,12 @@ describe('selectTests', () => { expect(result.selected).toContain('plan-ceo-review-selective'); expect(result.selected).toContain('plan-ceo-review-benefits'); expect(result.selected).toContain('plan-ceo-review-expansion-energy'); - expect(result.selected).toContain('autoplan-core'); expect(result.selected).toContain('codex-offered-ceo-review'); expect(result.selected).toContain('plan-ceo-review-format-mode'); expect(result.selected).toContain('plan-ceo-review-format-approach'); // v1.10.2.0 plan-mode handshake entries also depend on plan-ceo-review/** expect(result.selected).toContain('plan-ceo-review-plan-mode'); expect(result.selected).toContain('plan-mode-no-op'); - expect(result.selected).toContain('e2e-harness-audit'); expect(result.selected).toContain('plan-ceo-review-prosons-cadence'); expect(result.selected).toContain('plan-review-prosons-format'); expect(result.selected).toContain('plan-review-prosons-hardstop-neg'); @@ -115,8 +113,8 @@ describe('selectTests', () => { expect(result.selected).toContain('preamble-script-ab'); // AUQ repetition-cut NOT-WORSE gate drives plan-ceo-review, so it keys on it too. expect(result.selected).toContain('auq-repetition-cut-ab'); - expect(result.selected.length).toBe(25); - expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 25); + expect(result.selected.length).toBe(23); + expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 23); }); test('global touchfile triggers ALL tests', () => { @@ -388,3 +386,65 @@ describe('touchfile dependency paths exist', () => { } }); }); + +// --- Reverse invariant: every selection key names a LIVING test --- +// The forward invariants above catch stale dep PATHS; nothing caught stale +// KEYS. The 2026-08 audit found 15 phantom E2E keys (6 gate-tier) selecting +// tests that existed nowhere — the merge-blocking census counted work that +// could not run. A key earns its place by appearing as a quoted testName in +// a living paid test file; constructed names get a reasoned exception. + +describe('reverse invariant — keys must name living paid tests', () => { + const { isPaidTestFile } = require('./helpers/paid-test-set') as typeof import('./helpers/paid-test-set'); + + /** Keys whose testNames are CONSTRUCTED at runtime (template literals), so + * a quoted-occurrence scan cannot see them. Each entry needs the file that + * constructs it. Shrink-only: prefer literal names in new tests. */ + const CONSTRUCTED_NAME_EXCEPTIONS: Record = {}; + + const paidSources: string[] = []; + for (const name of fs.readdirSync(path.join(ROOT, 'test'))) { + const rel = `test/${name}`; + if (!isPaidTestFile(rel)) continue; + paidSources.push(fs.readFileSync(path.join(ROOT, rel), 'utf-8')); + } + + const quotedSomewhere = (key: string): boolean => + paidSources.some((src) => + src.includes(`'${key}'`) || src.includes(`"${key}"`) || src.includes('`' + key + '`')); + + /** Clause (b) of liveness: constructed testNames (template literals) bind + * through SELF-REGISTRATION — the 2026-08 dep-list sweep put each test + * FILE into its key's dep list, and the parent mapper keeps a shard on + * that registration union. So a key is alive when its name is quoted in a + * paid file OR its dep list names an existing paid test file. */ + const registeredToLivingFile = (key: string): boolean => + (E2E_TOUCHFILES[key] ?? []).some((dep) => + /\.test\.ts$/.test(dep) && isPaidTestFile(dep) && fs.existsSync(path.join(ROOT, dep))); + + test('every E2E_TOUCHFILES key is declared in a living paid test file', () => { + expect(paidSources.length).toBeGreaterThan(50); // scan-rot guard + const phantoms = Object.keys(E2E_TOUCHFILES) + .filter((key) => !(key in CONSTRUCTED_NAME_EXCEPTIONS)) + .filter((key) => !quotedSomewhere(key) && !registeredToLivingFile(key)); + expect( + phantoms, + `E2E_TOUCHFILES key(s) with NO declaring paid test — the census counts tests that cannot run. ` + + `Delete the key (both maps) or implement the test:\n ${phantoms.join('\n ')}`, + ).toEqual([]); + }); + + test('every LLM_JUDGE_TOUCHFILES key is declared in a living paid test file', () => { + const phantoms = Object.keys(LLM_JUDGE_TOUCHFILES).filter((key) => !quotedSomewhere(key)); + expect( + phantoms, + `LLM_JUDGE_TOUCHFILES key(s) with NO declaring test:\n ${phantoms.join('\n ')}`, + ).toEqual([]); + }); + + test('constructed-name exceptions stay live (files exist and construct them)', () => { + const stale = Object.entries(CONSTRUCTED_NAME_EXCEPTIONS) + .filter(([, file]) => !fs.existsSync(path.join(ROOT, file))); + expect(stale.map(([k]) => k), 'exception points at a deleted file — remove the entry').toEqual([]); + }); +});