From 9fbd0700ff7b1a2ad9ac129501d7c43e099e1945 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 04:37:37 +0000 Subject: [PATCH] fix(ci): kill the three zero-test eval jobs (hollow green) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - delete the vestigial e2e-codex / e2e-gemini matrix rows: both files are whole-file periodic-tier, so with no row tier: they ran ZERO tests and reported green on every PR (~2 min of runner each, pure false confidence; the periodic lane owns those suites) - e2e-pty-plan-smoke gains tier: gate — its two files are whole-file describeE2ETier('gate'), so the job burned ~7 min of container setup then skipped every describe - KNOWN_TIER_UNSET burned down to empty; the ratchet stays armed so a future row/file tier mismatch fails the suite instead of shipping hollow green Co-Authored-By: Claude Fable 5 --- .github/workflows/evals.yml | 12 ++++++++---- test/evals-workflow-matrix.test.ts | 22 +++++----------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index cfe7cbc0b..3aab49708 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -148,10 +148,10 @@ jobs: tier: gate - name: e2e-routing file: test/skill-routing-e2e.test.ts - - name: e2e-codex - file: test/codex-e2e.test.ts - - name: e2e-gemini - file: test/gemini-e2e.test.ts + # (e2e-codex / e2e-gemini rows deleted: both files are whole-file + # periodic-tier, so with no row tier: they ran ZERO tests and + # reported green on every PR — ~2 min of runner per PR of pure + # false confidence. The periodic lane owns these suites.) # Real-PTY plan-mode smokes. Only the deterministically-reliable ones # are CI-gated: office-hours (asks its mode question first, caught by # the collapsed/bullet prose-AUQ detector) and plan-mode-no-op (no @@ -161,6 +161,10 @@ jobs: # wedge on the fresh-container onboarding/API-key dialog. - name: e2e-pty-plan-smoke file: test/skill-e2e-office-hours-auto-mode.test.ts test/skill-e2e-plan-mode-no-op.test.ts + # Both files are whole-file describeE2ETier('gate') — without this + # row tier: the job burned ~7 min of setup then skipped every + # describe (hollow-green since the files adopted the self-gate). + tier: gate timeout: 35 # The documented contention-heavy PTY family: ROTATING members # failed attempt 2 in consecutive PR #2593 rounds diff --git a/test/evals-workflow-matrix.test.ts b/test/evals-workflow-matrix.test.ts index 5ae3f8958..53aeb67fa 100644 --- a/test/evals-workflow-matrix.test.ts +++ b/test/evals-workflow-matrix.test.ts @@ -52,24 +52,12 @@ const KNOWN_MATRIX_GAPS = new Set([ /** * Matrix files whose whole-file tier guard has no matching row `tier:` - * property (pre-existing, found 2026-08-26). Consequences today: - * - codex-e2e / gemini-e2e declare 'periodic' → both jobs run ZERO tests and - * report green on every PR (vestigial rows; the periodic cron lane owns - * these suites). - * - the two PTY plan-mode smokes declare 'gate' → the e2e-pty-plan-smoke job - * spends ~7 min on container setup and skill registration, then bun test - * skips every describe — hollow-green since the files adopted - * describeE2ETier. - * Fixing either means deliberately (re)activating paid suites on every PR — - * tracked in the same TODOS burn-down. Fix = add `tier:` to the row (or - * delete the vestigial row), then DELETE the entry here. + * property. Burned down to empty 2026-08-29: the vestigial codex/gemini rows + * were deleted (periodic-tier files, zero tests per PR) and + * e2e-pty-plan-smoke gained its `tier: gate`. The ratchet stays so a future + * row/file tier mismatch fails the suite instead of shipping hollow green. */ -const KNOWN_TIER_UNSET = new Map([ - ['test/codex-e2e.test.ts', 'periodic'], - ['test/gemini-e2e.test.ts', 'periodic'], - ['test/skill-e2e-office-hours-auto-mode.test.ts', 'gate'], - ['test/skill-e2e-plan-mode-no-op.test.ts', 'gate'], -]); +const KNOWN_TIER_UNSET = new Map([]); interface MatrixRow { name: string;