fix(evals): activate the 4 paid test files that could never run anywhere

carve-section-loading, codex-e2e-plan-format,
codex-e2e-recommendation-substance, and llm-judge-recommendation gated
on EVALS/tier (free suite loads them as describe.skip) but their names
fell outside PAID_TEST_GLOBS, so no paid lane ever selected them — net
execution zero, forever. The existing matrix tripwire filtered on
isPaidTestFile() first, so it was blind to exactly this class (the same
bug that hid the pre-split monolith's gate tests for ~8 releases).

- PAID_TEST_GLOBS: codex-e2e* + skill-llm-eval* wildcards (replacing
  exact names) + llm-judge-recommendation + carve-section-loading;
  package.json's six test-script glob lists mirrored
- codex-e2e-plan-format gains the explicit periodic tier gate its
  siblings carry (external-service rule) — without it the sharded
  runner's no-guard default would spawn Codex in the gate tier per PR
- eval:bg:periodic --timeout 32400→37800: the census growth pushed the
  periodic worst case to 35910s; the old value had 270s of headroom
  BEFORE this change and would now kill healthy runs mid-flight
- new test/paid-orphan-tripwire.test.ts: any EVALS/tier-gated test file
  outside the globs fails the free suite (reasoned SCANNER_EXEMPT for
  the gate helpers + meta-tests) — the class-killer
- paid-shards pins updated: the four orphans now assert INSIDE the
  census

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

package.json

@@ -22,12 +22,12 @@
     "test": "bun run scripts/test-free-shards.ts && (bun run slop:diff 2>/dev/null || true)",
     "test:free": "bun run scripts/test-free-shards.ts",
     "test:windows": "bun run scripts/test-free-shards.ts --windows-only",
-    "test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
-    "test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
-    "test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts",
-    "test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts",
-    "test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
-    "test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
+    "test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
     "test:gate:sharded": "bun run scripts/test-paid-shards.ts --tier gate",
     "test:periodic:sharded": "EVALS_ALL=1 bun run scripts/test-paid-shards.ts --tier periodic",
     "test:codex": "EVALS=1 bun test test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts",
@@ -40,7 +40,7 @@
     "eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals",
     "eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all",
     "eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded",
-    "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 32400 -- bun run test:periodic:sharded",
+    "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 37800 -- bun run test:periodic:sharded",
     "eval:list": "bun run scripts/eval-list.ts",
     "eval:compare": "bun run scripts/eval-compare.ts",
     "eval:summary": "bun run scripts/eval-summary.ts",

test/codex-e2e-plan-format.test.ts

@@ -47,7 +47,12 @@ const CODEX_AVAILABLE = (() => {
   } catch { return false; }
 })();
 const evalsEnabled = !!process.env.EVALS;
-const SKIP = !CODEX_AVAILABLE || !evalsEnabled;
+// External-service test — periodic tier only (CLAUDE.md tiering rule 3),
+// matching codex-e2e.test.ts / codex-e2e-sol-scope.test.ts. Without this
+// guard the sharded runner's "no whole-file tier guard" default would run
+// Codex spawns in the GATE tier on every PR.
+const tierOk = process.env.EVALS_TIER === 'periodic';
+const SKIP = !CODEX_AVAILABLE || !evalsEnabled || !tierOk;
 const describeCodex = SKIP ? describe.skip : describe;
 
 // --- Touchfiles ---

test/helpers/paid-test-set.ts

@@ -11,12 +11,20 @@ import { matchGlob } from './touchfiles';
 
 /** The exact globs package.json's `test:gate` passes to `bun test`. */
 export const PAID_TEST_GLOBS = [
-  'test/skill-llm-eval.test.ts',
+  // skill-llm-eval* (not just the base file): skill-llm-eval-spec.test.ts
+  // fell outside the exact glob and could never run in any lane.
+  'test/skill-llm-eval*.test.ts',
   'test/skill-e2e-*.test.ts',
   'test/skill-routing-e2e.test.ts',
-  'test/codex-e2e.test.ts',
+  // codex-e2e* (was two exact names): codex-e2e-plan-format.test.ts and
-  'test/codex-e2e-sol-scope.test.ts',
+  // codex-e2e-recommendation-substance.test.ts were API-spending orphans —
+  // outside these globs they self-skipped in the free suite AND never
+  // entered the paid census. The same bug class as the deleted pre-split
+  // monolith (see test/paid-shards.test.ts's regression pin).
+  'test/codex-e2e*.test.ts',
   'test/gemini-e2e.test.ts',
+  'test/llm-judge-recommendation.test.ts',
+  'test/carve-section-loading.test.ts',
 ] as const;
 
 /** True when a repo-relative path (either slash style) is a paid test file. */

test/paid-orphan-tripwire.test.ts

@@ -0,0 +1,80 @@
+/**
+ * No paid-gated test file may sit outside PAID_TEST_GLOBS.
+ *
+ * The orphan class this kills (found 2026-08): a file whose source gates on
+ * EVALS/tier (so the free suite loads it as describe.skip) but whose NAME
+ * doesn't match the paid globs (so no paid lane ever selects it) can never
+ * execute anywhere — forever, silently. Four files were in that state
+ * (codex-e2e-plan-format, codex-e2e-recommendation-substance,
+ * llm-judge-recommendation, carve-section-loading), and the tripwire built
+ * for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see
+ * them because it filters on isPaidTestFile() FIRST.
+ *
+ * Detection is over source text, so meta-tests and helpers that mention the
+ * gate patterns need reasoned exemptions (same convention as
+ * test/egress-receipt-wiring.test.ts's SCANNER_EXEMPT).
+ */
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { isPaidTestFile } from './helpers/paid-test-set';
+
+const ROOT = path.resolve(__dirname, '..');
+
+/** Files that legitimately mention gate patterns without being paid tests. */
+const SCANNER_EXEMPT = new Map<string, string>([
+  // The gate helpers themselves and their free unit tests:
+  ['test/helpers/e2e-gate.ts', 'defines the gate predicates'],
+  // Meta-tests that quote gate-pattern strings to test classification:
+  ['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'],
+  ['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'],
+  ['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'],
+  ['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'],
+  ['test/paid-orphan-tripwire.test.ts', 'this scanner'],
+]);
+
+/**
+ * Source shapes that mean "this file self-gates on the paid env":
+ * the shared helpers, or a direct EVALS/EVALS_TIER env read.
+ */
+const GATE_PATTERNS = [
+  /\bdescribeE2ETier\s*\(/,
+  /\be2eTierEnabled\s*\(/,
+  /process\.env\.EVALS\b/,
+];
+
+function trackedTestFiles(): string[] {
+  const out = spawnSync('git', ['ls-files', '*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
+  if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
+  return out.stdout.split('\n').filter(Boolean);
+}
+
+describe('paid orphan tripwire', () => {
+  test('every EVALS/tier-gated test file is inside PAID_TEST_GLOBS (or exempt with a reason)', () => {
+    const files = trackedTestFiles();
+    expect(files.length).toBeGreaterThan(100); // scan-rot guard
+
+    const orphans: string[] = [];
+    for (const rel of files) {
+      if (isPaidTestFile(rel)) continue;
+      if (SCANNER_EXEMPT.has(rel)) continue;
+      const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
+      const hit = GATE_PATTERNS.find((p) => p.test(source));
+      if (hit) orphans.push(`${rel} (matches ${hit})`);
+    }
+    expect(orphans,
+      'paid-gated test files OUTSIDE the paid globs can never run in any lane. '
+      + 'Fix: extend PAID_TEST_GLOBS in test/helpers/paid-test-set.ts (and mirror '
+      + 'package.json), or add a reasoned SCANNER_EXEMPT entry if the file only '
+      + `mentions the patterns:\n${orphans.join('\n')}`,
+    ).toEqual([]);
+  });
+
+  test('exemption entries stay real (stale entries must be deleted)', () => {
+    for (const [rel] of SCANNER_EXEMPT) {
+      expect(fs.existsSync(path.join(ROOT, rel)), `stale SCANNER_EXEMPT entry: ${rel}`).toBe(true);
+    }
+  });
+});

test/paid-shards.test.ts

@@ -43,15 +43,22 @@ describe('paid test enumeration', () => {
     // kept here as a regression pin: its glob-invisibility is exactly how
     // two gate tests went unexecuted for ~8 releases before the rehoming.
     expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
-    expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
     expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
+    // The 2026-08 orphan fix: these four were API-spending files OUTSIDE the
+    // globs — self-skipping in the free suite and absent from the paid
+    // census, so they could never run in any lane.
+    expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/codex-e2e-plan-format.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/llm-judge-recommendation.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/carve-section-loading.test.ts')).toBe(true);
+    expect(isPaidTestFile('test/skill-llm-eval-spec.test.ts')).toBe(true);
   });
 
   test('discovers files and gives each one its own shard', () => {
     const files = collectPaidTestFiles();
     expect(files.length).toBeGreaterThan(0);
     expect(files.every(isPaidTestFile)).toBe(true);
-    expect(PAID_TEST_GLOBS.length).toBe(6);
+    expect(PAID_TEST_GLOBS.length).toBe(7);
 
     const shards = planPaidShards(files);
     expect(shards.flat().sort()).toEqual([...files].sort());