diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index f30d651c4..4ba9baadf 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -59,7 +59,7 @@ export const PR_PROFILE_FILES: Record = { export interface PrProfileMaps { e2eTouchfiles: Record; judgeTouchfiles: Record; - tiers: Record; + tiers: Record; globalTouchfiles: readonly string[]; } @@ -72,7 +72,7 @@ export interface PrProfileSelection { mode: 'pr' | 'full-fallback'; e2e: string[]; judges: string[]; - deferred: Array<{ id: string; tier: 'gate' | 'periodic'; reason: string }>; + deferred: Array<{ id: string; tier: 'gate' | 'periodic' | 'marathon'; reason: string }>; unknownFiles: string[]; deferredPromptFiles: string[]; missingCoverage: string[]; @@ -92,8 +92,8 @@ export function validatePrProfileInventory( } } for (const id of Object.keys(maps.e2eTouchfiles)) { - if (maps.tiers[id] !== 'gate' && maps.tiers[id] !== 'periodic') { - throw new Error(`E2E case has no broad gate/periodic census: ${id}`); + if (maps.tiers[id] !== 'gate' && maps.tiers[id] !== 'periodic' && maps.tiers[id] !== 'marathon') { + throw new Error(`E2E case has no broad gate/periodic/marathon census: ${id}`); } } } @@ -182,9 +182,11 @@ export function selectPrProfile(options: { const kept = new Set(e2e); const deferred = candidates.filter(id => !kept.has(id)).map(id => ({ id, tier: maps.tiers[id], - reason: maps.tiers[id] === 'periodic' - ? 'Broad periodic/release coverage; not executed by the PR gate' - : 'Broad gate census/release coverage; outside the fast PR profile', + reason: maps.tiers[id] === 'marathon' + ? 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate' + : maps.tiers[id] === 'periodic' + ? 'Broad periodic/release coverage; not executed by the PR gate' + : 'Broad gate census/release coverage; outside the fast PR profile', })); const noQuickCoverage = files.filter(file => isPromptFile(file) && !(depends(file, maps.globalTouchfiles) && (e2e.length > 0 || judges.length > 0)) diff --git a/test/helpers/e2e-gate.ts b/test/helpers/e2e-gate.ts index 5bd147e7d..32f482be4 100644 --- a/test/helpers/e2e-gate.ts +++ b/test/helpers/e2e-gate.ts @@ -24,11 +24,12 @@ import { describe } from 'bun:test'; -export type E2ETier = 'gate' | 'periodic'; +export type E2ETier = 'gate' | 'periodic' | 'marathon'; /** * True when this process should run whole-file-gated paid tests of `tier`: - * EVALS=1 AND EVALS_TIER exactly equals the tier. + * EVALS=1 AND EVALS_TIER exactly equals the tier. 'marathon' cases (full + * end-to-end flows) therefore never run in the gate/PR or periodic lanes. * * Deliberate consequence: EVALS=1 with EVALS_TIER unset is false for BOTH * tiers. Tierless runs (`test:evals` / `eval:bg` / `eval:bg:all`) skip every diff --git a/test/helpers/e2e-gate.unit.test.ts b/test/helpers/e2e-gate.unit.test.ts index 418eb06be..79c06908a 100644 --- a/test/helpers/e2e-gate.unit.test.ts +++ b/test/helpers/e2e-gate.unit.test.ts @@ -64,6 +64,19 @@ describe('e2e-gate: env matrix (read at call time)', () => { expect(describeE2ETier('periodic')).toBe(describe.skip); }); + test('marathon runs only in its own lane; gate and periodic lanes skip it', () => { + process.env.EVALS = '1'; + for (const lane of ['gate', 'periodic']) { + process.env.EVALS_TIER = lane; + expect(e2eTierEnabled('marathon')).toBe(false); + expect(describeE2ETier('marathon')).toBe(describe.skip); + } + process.env.EVALS_TIER = 'marathon'; + expect(describeE2ETier('marathon')).toBe(describe); + expect(describeE2ETier('gate')).toBe(describe.skip); + expect(describeE2ETier('periodic')).toBe(describe.skip); + }); + test('EVALS=1 + EVALS_TIER unset → skip both tiers (the tierless test:evals / eval:bg:all trap)', () => { process.env.EVALS = '1'; expect(e2eTierEnabled('gate')).toBe(false); diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 443040ed6..63ba664cb 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -116,9 +116,10 @@ export let selectedTests: string[] | null = resolveModuleSelection( // EVALS_TIER: filter tests by tier after diff-based selection. // 'gate' = gate tests only (CI default — blocks merge) // 'periodic' = periodic tests only (weekly cron / manual) +// 'marathon' = full end-to-end flows only (non-blocking marathon lane) // not set = run all selected tests (local dev default, backward compat) if (evalsEnabled && process.env.EVALS_TIER) { - const tier = process.env.EVALS_TIER as 'gate' | 'periodic'; + const tier = process.env.EVALS_TIER as 'gate' | 'periodic' | 'marathon'; const tierTests = Object.entries(E2E_TIERS) .filter(([, t]) => t === tier) .map(([name]) => name); diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 9094ee72d..0a0780804 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1118,10 +1118,11 @@ export const E2E_TOUCHFILES: Record = { }; /** - * E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand. + * E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand, + * 'marathon' keeps full start-to-finish flows in a non-blocking lane only. * Must have exactly the same keys as E2E_TOUCHFILES. */ -export const E2E_TIERS: Record = { +export const E2E_TIERS: Record = { 'ship-skipped-queued-finding': 'gate', 'investigate-owned-completion': 'gate', 'investigate-owned-abort': 'gate', diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index 975990a62..a9a2d99cb 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -46,7 +46,7 @@ if (evalsEnabled && !process.env.EVALS_ALL) { // Apply EVALS_TIER filter (same logic as e2e-helpers.ts) if (evalsEnabled && process.env.EVALS_TIER) { - const tier = process.env.EVALS_TIER as 'gate' | 'periodic'; + const tier = process.env.EVALS_TIER as 'gate' | 'periodic' | 'marathon'; const tierTests = Object.entries(E2E_TIERS) .filter(([, t]) => t === tier) .map(([name]) => name); diff --git a/test/test-pr-profile.test.ts b/test/test-pr-profile.test.ts index 941635fdb..2ec5ec0c7 100644 --- a/test/test-pr-profile.test.ts +++ b/test/test-pr-profile.test.ts @@ -35,7 +35,7 @@ describe('fast PR coverage policy', () => { expect(PR_PROFILE_CASE_IDS).toContain('plan-ceo-review-benefits'); expect(PR_PROFILE_CASE_IDS).toContain('plan-review-report'); expect(PR_PROFILE_CASE_IDS).toContain('auq-format-gate'); - for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic']).toContain(tier); + for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic', 'marathon']).toContain(tier); }); test('keeps relevant short probes and the failing CEO quality obligation; reports broad work', () => { @@ -151,6 +151,17 @@ describe('fast PR coverage policy', () => { expect(result.needsFullValidation).toBe(false); }); + test('marathon cases are deferred to their non-blocking lane, even on full fallback', () => { + const marathon: PrProfileMaps = { ...maps, tiers: { ...maps.tiers, 'ceo-full': 'marathon' } }; + for (const changedFiles of [['plan-ceo-review/SKILL.md.tmpl'], ['lib/new-runtime.ts']]) { + const result = select({ maps: marathon, changedFiles }); + expect(result.e2e).not.toContain('ceo-full'); + expect(result.deferred.find(({ id }) => id === 'ceo-full')).toEqual({ id: 'ceo-full', tier: 'marathon', + reason: 'Full end-to-end marathon coverage; non-blocking lane, not executed by the PR gate' }); + } + expect(() => select({ maps: marathon, profile: ['ceo-full'] })).toThrow('broad gate census'); + }); + test('rejects stale profile IDs, incorrect tiers, missing cadence, and unknown selections', () => { expect(() => select({ profile: ['missing'] })).toThrow('broad gate census'); expect(() => select({ profile: ['ceo-periodic'] })).toThrow('broad gate census');