From a41cdb7d7def399bc04856b44189a05b521cfe19 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 29 Sep 2026 15:21:17 +0000 Subject: [PATCH] test: add the non-blocking 'marathon' E2E tier Full start-to-finish flows move out of the blocking lanes. E2E_TIERS and E2ETier gain 'marathon'; describeE2ETier('marathon') is enabled only when EVALS_TIER=marathon, so the gate/PR and periodic lanes (and the gate census) never run those cases. The PR profile accepts marathon ids as scheduled elsewhere and defers them with their own reason, even on full fallback. --- scripts/test-pr-profile.ts | 16 +++++++++------- test/helpers/e2e-gate.ts | 5 +++-- test/helpers/e2e-gate.unit.test.ts | 13 +++++++++++++ test/helpers/e2e-helpers.ts | 3 ++- test/helpers/touchfiles-data.ts | 5 +++-- test/skill-routing-e2e.test.ts | 2 +- test/test-pr-profile.test.ts | 13 ++++++++++++- 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index f30d651c4..4ba9baadf 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -59,7 +59,7 @@ export const PR_PROFILE_FILES: Record = { export interface PrProfileMaps { e2eTouchfiles: Record; judgeTouchfiles: Record; - tiers: Record; + tiers: Record; globalTouchfiles: readonly string[]; } @@ -72,7 +72,7 @@ export interface PrProfileSelection { mode: 'pr' | 'full-fallback'; e2e: string[]; judges: string[]; - deferred: Array<{ id: string; tier: 'gate' | 'periodic'; reason: string }>; + deferred: Array<{ id: string; tier: 'gate' | 'periodic' | 'marathon'; reason: string }>; unknownFiles: string[]; deferredPromptFiles: string[]; missingCoverage: string[]; @@ -92,8 +92,8 @@ export function validatePrProfileInventory( } } for (const id of Object.keys(maps.e2eTouchfiles)) { - if (maps.tiers[id] !== 'gate' && maps.tiers[id] !== 'periodic') { - throw new Error(`E2E case has no broad gate/periodic census: ${id}`); + if (maps.tiers[id] !== 'gate' && maps.tiers[id] !== 'periodic' && maps.tiers[id] !== 'marathon') { + throw new Error(`E2E case has no broad gate/periodic/marathon census: ${id}`); } } } @@ -182,9 +182,11 @@ export function selectPrProfile(options: { const kept = new Set(e2e); const deferred = candidates.filter(id => !kept.has(id)).map(id => ({ id, tier: maps.tiers[id], - reason: maps.tiers[id] === 'periodic' - ? 'Broad periodic/release coverage; not executed by the PR gate' - : 'Broad gate census/release coverage; outside the fast PR profile', + reason: maps.tiers[id] === 'marathon' + ? 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate' + : maps.tiers[id] === 'periodic' + ? 'Broad periodic/release coverage; not executed by the PR gate' + : 'Broad gate census/release coverage; outside the fast PR profile', })); const noQuickCoverage = files.filter(file => isPromptFile(file) && !(depends(file, maps.globalTouchfiles) && (e2e.length > 0 || judges.length > 0)) diff --git a/test/helpers/e2e-gate.ts b/test/helpers/e2e-gate.ts index 5bd147e7d..32f482be4 100644 --- a/test/helpers/e2e-gate.ts +++ b/test/helpers/e2e-gate.ts @@ -24,11 +24,12 @@ import { describe } from 'bun:test'; -export type E2ETier = 'gate' | 'periodic'; +export type E2ETier = 'gate' | 'periodic' | 'marathon'; /** * True when this process should run whole-file-gated paid tests of `tier`: - * EVALS=1 AND EVALS_TIER exactly equals the tier. + * EVALS=1 AND EVALS_TIER exactly equals the tier. 'marathon' cases (full + * end-to-end flows) therefore never run in the gate/PR or periodic lanes. * * Deliberate consequence: EVALS=1 with EVALS_TIER unset is false for BOTH * tiers. Tierless runs (`test:evals` / `eval:bg` / `eval:bg:all`) skip every diff --git a/test/helpers/e2e-gate.unit.test.ts b/test/helpers/e2e-gate.unit.test.ts index 418eb06be..79c06908a 100644 --- a/test/helpers/e2e-gate.unit.test.ts +++ b/test/helpers/e2e-gate.unit.test.ts @@ -64,6 +64,19 @@ describe('e2e-gate: env matrix (read at call time)', () => { expect(describeE2ETier('periodic')).toBe(describe.skip); }); + test('marathon runs only in its own lane; gate and periodic lanes skip it', () => { + process.env.EVALS = '1'; + for (const lane of ['gate', 'periodic']) { + process.env.EVALS_TIER = lane; + expect(e2eTierEnabled('marathon')).toBe(false); + expect(describeE2ETier('marathon')).toBe(describe.skip); + } + process.env.EVALS_TIER = 'marathon'; + expect(describeE2ETier('marathon')).toBe(describe); + expect(describeE2ETier('gate')).toBe(describe.skip); + expect(describeE2ETier('periodic')).toBe(describe.skip); + }); + test('EVALS=1 + EVALS_TIER unset → skip both tiers (the tierless test:evals / eval:bg:all trap)', () => { process.env.EVALS = '1'; expect(e2eTierEnabled('gate')).toBe(false); diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 443040ed6..63ba664cb 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -116,9 +116,10 @@ export let selectedTests: string[] | null = resolveModuleSelection( // EVALS_TIER: filter tests by tier after diff-based selection. // 'gate' = gate tests only (CI default — blocks merge) // 'periodic' = periodic tests only (weekly cron / manual) +// 'marathon' = full end-to-end flows only (non-blocking marathon lane) // not set = run all selected tests (local dev default, backward compat) if (evalsEnabled && process.env.EVALS_TIER) { - const tier = process.env.EVALS_TIER as 'gate' | 'periodic'; + const tier = process.env.EVALS_TIER as 'gate' | 'periodic' | 'marathon'; const tierTests = Object.entries(E2E_TIERS) .filter(([, t]) => t === tier) .map(([name]) => name); diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 9094ee72d..0a0780804 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1118,10 +1118,11 @@ export const E2E_TOUCHFILES: Record = { }; /** - * E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand. + * E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand, + * 'marathon' keeps full start-to-finish flows in a non-blocking lane only. * Must have exactly the same keys as E2E_TOUCHFILES. */ -export const E2E_TIERS: Record = { +export const E2E_TIERS: Record = { 'ship-skipped-queued-finding': 'gate', 'investigate-owned-completion': 'gate', 'investigate-owned-abort': 'gate', diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index 975990a62..a9a2d99cb 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -46,7 +46,7 @@ if (evalsEnabled && !process.env.EVALS_ALL) { // Apply EVALS_TIER filter (same logic as e2e-helpers.ts) if (evalsEnabled && process.env.EVALS_TIER) { - const tier = process.env.EVALS_TIER as 'gate' | 'periodic'; + const tier = process.env.EVALS_TIER as 'gate' | 'periodic' | 'marathon'; const tierTests = Object.entries(E2E_TIERS) .filter(([, t]) => t === tier) .map(([name]) => name); diff --git a/test/test-pr-profile.test.ts b/test/test-pr-profile.test.ts index 941635fdb..2ec5ec0c7 100644 --- a/test/test-pr-profile.test.ts +++ b/test/test-pr-profile.test.ts @@ -35,7 +35,7 @@ describe('fast PR coverage policy', () => { expect(PR_PROFILE_CASE_IDS).toContain('plan-ceo-review-benefits'); expect(PR_PROFILE_CASE_IDS).toContain('plan-review-report'); expect(PR_PROFILE_CASE_IDS).toContain('auq-format-gate'); - for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic']).toContain(tier); + for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic', 'marathon']).toContain(tier); }); test('keeps relevant short probes and the failing CEO quality obligation; reports broad work', () => { @@ -151,6 +151,17 @@ describe('fast PR coverage policy', () => { expect(result.needsFullValidation).toBe(false); }); + test('marathon cases are deferred to their non-blocking lane, even on full fallback', () => { + const marathon: PrProfileMaps = { ...maps, tiers: { ...maps.tiers, 'ceo-full': 'marathon' } }; + for (const changedFiles of [['plan-ceo-review/SKILL.md.tmpl'], ['lib/new-runtime.ts']]) { + const result = select({ maps: marathon, changedFiles }); + expect(result.e2e).not.toContain('ceo-full'); + expect(result.deferred.find(({ id }) => id === 'ceo-full')).toEqual({ id: 'ceo-full', tier: 'marathon', + reason: 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate' }); + } + expect(() => select({ maps: marathon, profile: ['ceo-full'] })).toThrow('broad gate census'); + }); + test('rejects stale profile IDs, incorrect tiers, missing cadence, and unknown selections', () => { expect(() => select({ profile: ['missing'] })).toThrow('broad gate census'); expect(() => select({ profile: ['ceo-periodic'] })).toThrow('broad gate census');