From bd5935864031449b31d84ce012a49a6ac25d7e4e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 25 Aug 2026 20:50:15 +0000 Subject: [PATCH] test(ci): demote the preamble A/B to periodic (OV7) and add it to the periodic matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-Phase-3 demotion per the plan; the eval needs fetch-depth 0 (it git shows a pre-Phase-1 sha), which only the periodic workflow provides — and a static matrix entry so it can't silently never run. Co-Authored-By: Claude Fable 5 --- .github/workflows/evals-periodic.yml | 2 ++ test/helpers/touchfiles-data.ts | 2 +- test/skill-e2e-preamble-script-ab.test.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 3017868b9..00510d640 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -91,6 +91,8 @@ jobs: file: test/skill-e2e-review.test.ts - name: e2e-retro file: test/skill-e2e-retro.test.ts + - name: e2e-preamble-ab + file: test/skill-e2e-preamble-script-ab.test.ts # e2e-review-attribution, e2e-coverage-audit, and e2e-triage are # gate-only (every test they hold is gate-tier) — deliberately absent # here; an all-skip shard would just burn a container boot weekly. diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index cc89724f6..3c336919e 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -540,7 +540,7 @@ export const E2E_TIERS: Record = { // Real-PTY E2E batch — tier classification: // gate: cheap, deterministic, run on every PR // periodic: long-running or expensive (>$3/run), run weekly - 'preamble-script-ab': 'gate', // Phase 1-3 A/B: script vs inline preamble; demote to periodic after Phase 3 (OV7) + 'preamble-script-ab': 'periodic', // Phase 1-3 A/B: script vs inline preamble; demoted post-Phase-3 (OV7) 'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe 'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions 'plan-design-with-ui-scope': 'gate', // ~$0.80/run diff --git a/test/skill-e2e-preamble-script-ab.test.ts b/test/skill-e2e-preamble-script-ab.test.ts index c3ba57022..7b0827c59 100644 --- a/test/skill-e2e-preamble-script-ab.test.ts +++ b/test/skill-e2e-preamble-script-ab.test.ts @@ -33,7 +33,7 @@ import { carvedSkill, } from './helpers/auq-sdk-capture'; -const describeE2E = describeE2ETier('gate'); +const describeE2E = describeE2ETier('periodic'); const runId = `preamble-ab-${process.env.EVALS_RUN_ID ?? 'local'}`; const ROOT = path.resolve(import.meta.dir, '..'); const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)