From 72a5246aae725c2790107472fe9030d5d64a9fb2 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 04:52:47 +0000 Subject: [PATCH] fix(evals): activate the 4 paid test files that could never run anywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit carve-section-loading, codex-e2e-plan-format, codex-e2e-recommendation-substance, and llm-judge-recommendation gated on EVALS/tier (free suite loads them as describe.skip) but their names fell outside PAID_TEST_GLOBS, so no paid lane ever selected them — net execution zero, forever. The existing matrix tripwire filtered on isPaidTestFile() first, so it was blind to exactly this class (the same bug that hid the pre-split monolith's gate tests for ~8 releases). - PAID_TEST_GLOBS: codex-e2e* + skill-llm-eval* wildcards (replacing exact names) + llm-judge-recommendation + carve-section-loading; package.json's six test-script glob lists mirrored - codex-e2e-plan-format gains the explicit periodic tier gate its siblings carry (external-service rule) — without it the sharded runner's no-guard default would spawn Codex in the gate tier per PR - eval:bg:periodic --timeout 32400→37800: the census growth pushed the periodic worst case to 35910s; the old value had 270s of headroom BEFORE this change and would now kill healthy runs mid-flight - new test/paid-orphan-tripwire.test.ts: any EVALS/tier-gated test file outside the globs fails the free suite (reasoned SCANNER_EXEMPT for the gate helpers + meta-tests) — the class-killer - paid-shards pins updated: the four orphans now assert INSIDE the census Co-Authored-By: Claude Fable 5 --- package.json | 14 +++--- test/codex-e2e-plan-format.test.ts | 7 ++- test/helpers/paid-test-set.ts | 14 ++++-- test/paid-orphan-tripwire.test.ts | 80 ++++++++++++++++++++++++++++++ test/paid-shards.test.ts | 11 +++- 5 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 test/paid-orphan-tripwire.test.ts diff --git a/package.json b/package.json index dd20d28b6..2b94b0d60 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,12 @@ "test": "bun run scripts/test-free-shards.ts && (bun run slop:diff 2>/dev/null || true)", "test:free": "bun run scripts/test-free-shards.ts", "test:windows": "bun run scripts/test-free-shards.ts --windows-only", - "test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", - "test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", - "test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", - "test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", - "test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", - "test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts", + "test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts", + "test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts", + "test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts", + "test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts", + "test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts", + "test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts", "test:gate:sharded": "bun run scripts/test-paid-shards.ts --tier gate", "test:periodic:sharded": "EVALS_ALL=1 bun run scripts/test-paid-shards.ts --tier periodic", "test:codex": "EVALS=1 bun test test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts", @@ -40,7 +40,7 @@ "eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals", "eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all", "eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded", - "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 32400 -- bun run test:periodic:sharded", + "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 37800 -- bun run test:periodic:sharded", "eval:list": "bun run scripts/eval-list.ts", "eval:compare": "bun run scripts/eval-compare.ts", "eval:summary": "bun run scripts/eval-summary.ts", diff --git a/test/codex-e2e-plan-format.test.ts b/test/codex-e2e-plan-format.test.ts index 0481f69d9..acf92f72e 100644 --- a/test/codex-e2e-plan-format.test.ts +++ b/test/codex-e2e-plan-format.test.ts @@ -47,7 +47,12 @@ const CODEX_AVAILABLE = (() => { } catch { return false; } })(); const evalsEnabled = !!process.env.EVALS; -const SKIP = !CODEX_AVAILABLE || !evalsEnabled; +// External-service test — periodic tier only (CLAUDE.md tiering rule 3), +// matching codex-e2e.test.ts / codex-e2e-sol-scope.test.ts. Without this +// guard the sharded runner's "no whole-file tier guard" default would run +// Codex spawns in the GATE tier on every PR. +const tierOk = process.env.EVALS_TIER === 'periodic'; +const SKIP = !CODEX_AVAILABLE || !evalsEnabled || !tierOk; const describeCodex = SKIP ? describe.skip : describe; // --- Touchfiles --- diff --git a/test/helpers/paid-test-set.ts b/test/helpers/paid-test-set.ts index 6acb7cd00..956a1937f 100644 --- a/test/helpers/paid-test-set.ts +++ b/test/helpers/paid-test-set.ts @@ -11,12 +11,20 @@ import { matchGlob } from './touchfiles'; /** The exact globs package.json's `test:gate` passes to `bun test`. */ export const PAID_TEST_GLOBS = [ - 'test/skill-llm-eval.test.ts', + // skill-llm-eval* (not just the base file): skill-llm-eval-spec.test.ts + // fell outside the exact glob and could never run in any lane. + 'test/skill-llm-eval*.test.ts', 'test/skill-e2e-*.test.ts', 'test/skill-routing-e2e.test.ts', - 'test/codex-e2e.test.ts', - 'test/codex-e2e-sol-scope.test.ts', + // codex-e2e* (was two exact names): codex-e2e-plan-format.test.ts and + // codex-e2e-recommendation-substance.test.ts were API-spending orphans — + // outside these globs they self-skipped in the free suite AND never + // entered the paid census. The same bug class as the deleted pre-split + // monolith (see test/paid-shards.test.ts's regression pin). + 'test/codex-e2e*.test.ts', 'test/gemini-e2e.test.ts', + 'test/llm-judge-recommendation.test.ts', + 'test/carve-section-loading.test.ts', ] as const; /** True when a repo-relative path (either slash style) is a paid test file. */ diff --git a/test/paid-orphan-tripwire.test.ts b/test/paid-orphan-tripwire.test.ts new file mode 100644 index 000000000..b1f0bb1f6 --- /dev/null +++ b/test/paid-orphan-tripwire.test.ts @@ -0,0 +1,80 @@ +/** + * No paid-gated test file may sit outside PAID_TEST_GLOBS. + * + * The orphan class this kills (found 2026-08): a file whose source gates on + * EVALS/tier (so the free suite loads it as describe.skip) but whose NAME + * doesn't match the paid globs (so no paid lane ever selects it) can never + * execute anywhere — forever, silently. Four files were in that state + * (codex-e2e-plan-format, codex-e2e-recommendation-substance, + * llm-judge-recommendation, carve-section-loading), and the tripwire built + * for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see + * them because it filters on isPaidTestFile() FIRST. + * + * Detection is over source text, so meta-tests and helpers that mention the + * gate patterns need reasoned exemptions (same convention as + * test/egress-receipt-wiring.test.ts's SCANNER_EXEMPT). + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { isPaidTestFile } from './helpers/paid-test-set'; + +const ROOT = path.resolve(__dirname, '..'); + +/** Files that legitimately mention gate patterns without being paid tests. */ +const SCANNER_EXEMPT = new Map([ + // The gate helpers themselves and their free unit tests: + ['test/helpers/e2e-gate.ts', 'defines the gate predicates'], + // Meta-tests that quote gate-pattern strings to test classification: + ['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'], + ['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'], + ['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'], + ['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'], + ['test/paid-orphan-tripwire.test.ts', 'this scanner'], +]); + +/** + * Source shapes that mean "this file self-gates on the paid env": + * the shared helpers, or a direct EVALS/EVALS_TIER env read. + */ +const GATE_PATTERNS = [ + /\bdescribeE2ETier\s*\(/, + /\be2eTierEnabled\s*\(/, + /process\.env\.EVALS\b/, +]; + +function trackedTestFiles(): string[] { + const out = spawnSync('git', ['ls-files', '*.test.ts'], { cwd: ROOT, encoding: 'utf-8' }); + if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`); + return out.stdout.split('\n').filter(Boolean); +} + +describe('paid orphan tripwire', () => { + test('every EVALS/tier-gated test file is inside PAID_TEST_GLOBS (or exempt with a reason)', () => { + const files = trackedTestFiles(); + expect(files.length).toBeGreaterThan(100); // scan-rot guard + + const orphans: string[] = []; + for (const rel of files) { + if (isPaidTestFile(rel)) continue; + if (SCANNER_EXEMPT.has(rel)) continue; + const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8'); + const hit = GATE_PATTERNS.find((p) => p.test(source)); + if (hit) orphans.push(`${rel} (matches ${hit})`); + } + expect(orphans, + 'paid-gated test files OUTSIDE the paid globs can never run in any lane. ' + + 'Fix: extend PAID_TEST_GLOBS in test/helpers/paid-test-set.ts (and mirror ' + + 'package.json), or add a reasoned SCANNER_EXEMPT entry if the file only ' + + `mentions the patterns:\n${orphans.join('\n')}`, + ).toEqual([]); + }); + + test('exemption entries stay real (stale entries must be deleted)', () => { + for (const [rel] of SCANNER_EXEMPT) { + expect(fs.existsSync(path.join(ROOT, rel)), `stale SCANNER_EXEMPT entry: ${rel}`).toBe(true); + } + }); +}); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index f75e5d322..771d5b0b8 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -43,15 +43,22 @@ describe('paid test enumeration', () => { // kept here as a regression pin: its glob-invisibility is exactly how // two gate tests went unexecuted for ~8 releases before the rehoming. expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false); - expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false); expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false); + // The 2026-08 orphan fix: these four were API-spending files OUTSIDE the + // globs — self-skipping in the free suite and absent from the paid + // census, so they could never run in any lane. + expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(true); + expect(isPaidTestFile('test/codex-e2e-plan-format.test.ts')).toBe(true); + expect(isPaidTestFile('test/llm-judge-recommendation.test.ts')).toBe(true); + expect(isPaidTestFile('test/carve-section-loading.test.ts')).toBe(true); + expect(isPaidTestFile('test/skill-llm-eval-spec.test.ts')).toBe(true); }); test('discovers files and gives each one its own shard', () => { const files = collectPaidTestFiles(); expect(files.length).toBeGreaterThan(0); expect(files.every(isPaidTestFile)).toBe(true); - expect(PAID_TEST_GLOBS.length).toBe(6); + expect(PAID_TEST_GLOBS.length).toBe(7); const shards = planPaidShards(files); expect(shards.flat().sort()).toEqual([...files].sort());