CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-18 10:52:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(test): migrate the 36 tier-gated eval files to describeE2ETier

Browse Source 
Mechanical two-liner swap in 34 files (each keeping its declared tier — all
36 predicates verified against E2E_TIERS before migrating); the two files
with compound gates (overlay-harness's EvalCollector feed, codex-e2e's
CODEX_AVAILABLE) keep their extra conditions via e2eTierEnabled. Tier
rationale comments preserved. codex-e2e/gemini-e2e/benchmark-providers keep
their distinct stderr-message gate shapes by design.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-14 21:12:23 -07:00
co-authored by Claude Fable 5
parent e23a3ae3b1
commit 290c71046b
36 changed files with 108 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/carve-section-loading.test.ts
+3 -3
View File
@@ -19,12 +19,12 @@
* Step-0 mode loop) and keep their dedicated tests; E1 asserts those exist. * Step-0 mode loop) and keep their dedicated tests; E1 asserts those exist.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture'; import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
import { CARVE_GUARDS } from './helpers/carve-guards'; import { CARVE_GUARDS } from './helpers/carve-guards';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `carve-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `carve-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
const only = process.env.GSTACK_CARVE_SKILL?.trim(); const only = process.env.GSTACK_CARVE_SKILL?.trim();
test/codex-e2e-recommendation-substance.test.ts
+2 -2
View File
@@ -22,6 +22,7 @@
*/ */
import { describe, test, expect } from 'bun:test'; import { describe, test, expect } from 'bun:test';
import * as path from 'node:path'; import * as path from 'node:path';
import { e2eTierEnabled } from './helpers/e2e-gate';
import { runCodexSkill } from './helpers/codex-session-runner'; import { runCodexSkill } from './helpers/codex-session-runner';
import { judgeRecommendation } from './helpers/llm-judge'; import { judgeRecommendation } from './helpers/llm-judge';
@@ -34,8 +35,7 @@ const CODEX_AVAILABLE = (() => {
return false; return false;
} }
})(); })();
const shouldRun = const shouldRun = CODEX_AVAILABLE && e2eTierEnabled('periodic');
CODEX_AVAILABLE && !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
const describeCodex = shouldRun ? describe : describe.skip; const describeCodex = shouldRun ? describe : describe.skip;
// A small fixture with two real, comparable problems so a good recommendation // A small fixture with two real, comparable problems so a good recommendation
test/skill-e2e-ask-user-question-format-compliance.test.ts
+3 -3
View File
@@ -22,7 +22,8 @@
* hide; the model's composed question is. Shares the engine with the periodic * hide; the model's composed question is. Shares the engine with the periodic
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts). * A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
setupPlanCeoDir, setupPlanCeoDir,
@@ -32,8 +33,7 @@ import {
carvedSkill, carvedSkill,
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `auq-format-gate-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `auq-format-gate-${process.env.EVALS_RUN_ID ?? 'local'}`;
describeE2E('AskUserQuestion format compliance (gate)', () => { describeE2E('AskUserQuestion format compliance (gate)', () => {
test/skill-e2e-auq-consistency.test.ts
+3 -3
View File
@@ -15,7 +15,8 @@
* Reports per-run scores so drift is visible even on a pass. Periodic tier * Reports per-run scores so drift is visible even on a pass. Periodic tier
* (N SDK runs, ~$0.50-1 each). * (N SDK runs, ~$0.50-1 each).
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
setupPlanCeoDir, setupPlanCeoDir,
@@ -25,8 +26,7 @@ import {
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
import { judgeRecommendation } from './helpers/llm-judge'; import { judgeRecommendation } from './helpers/llm-judge';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3'); const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3');
const runId = `auq-consistency-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `auq-consistency-${process.env.EVALS_RUN_ID ?? 'local'}`;
test/skill-e2e-auq-matrix.test.ts
+3 -3
View File
@@ -22,7 +22,8 @@
* *
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso". * Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
setupSkillDir, setupSkillDir,
@@ -32,8 +33,7 @@ import {
gradeAuqRecommendation, gradeAuqRecommendation,
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`;
const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean); const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean);
test/skill-e2e-auq-verbose-vs-carved-ab.test.ts
+3 -3
View File
@@ -22,7 +22,8 @@
* carries the same {{PREAMBLE}} format spec + Step 0 prose as verbose, with * carries the same {{PREAMBLE}} format spec + Step 0 prose as verbose, with
* strictly less unrelated review-section text in context. * strictly less unrelated review-section text in context.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
setupPlanCeoDir, setupPlanCeoDir,
@@ -33,8 +34,7 @@ import {
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
import { judgeRecommendation } from './helpers/llm-judge'; import { judgeRecommendation } from './helpers/llm-judge';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `auq-ab-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `auq-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
async function grade(label: string, dir: string) { async function grade(label: string, dir: string) {
test/skill-e2e-auto-decide-preserved.test.ts
+3 -3
View File
@@ -37,15 +37,15 @@
* practice but not the load-bearing behavior). * practice but not the load-bearing behavior).
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const ROOT = path.resolve(import.meta.dir, '..'); const ROOT = path.resolve(import.meta.dir, '..');
test/skill-e2e-autoplan-chain.test.ts
+3 -3
View File
@@ -24,7 +24,8 @@
* Cost: ~$5-8/run, 10-15 min wall clock. Periodic — runs weekly. * Cost: ~$5-8/run, 10-15 min wall clock. Periodic — runs weekly.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
@@ -36,8 +37,7 @@ import {
isNumberedOptionListVisible, isNumberedOptionListVisible,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const ROOT = path.resolve(import.meta.dir, '..'); const ROOT = path.resolve(import.meta.dir, '..');
const UI_FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md'); const UI_FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
test/skill-e2e-brain-privacy-gate.test.ts
+3 -3
View File
@@ -20,14 +20,14 @@
* prose contract this test locks in. * prose contract this test locks in.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner'; import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('gbrain-sync privacy gate fires once via preamble', () => { describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
test('gstack skill preamble fires the 3-option AskUserQuestion when gbrain is detected', async () => { test('gstack skill preamble fires the 3-option AskUserQuestion when gbrain is detected', async () => {
test/skill-e2e-conductor-prose.test.ts
+3 -3
View File
@@ -20,11 +20,11 @@
* Periodic tier: model-behavior, non-deterministic. * Periodic tier: model-behavior, non-deterministic.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const FLAWED_PLAN = `# Plan: add a "developer-friendly" pricing tier const FLAWED_PLAN = `# Plan: add a "developer-friendly" pricing tier
test/skill-e2e-office-hours-auto-mode.test.ts
+3 -3
View File
@@ -16,11 +16,11 @@
* distinct silencing mechanism; both share the same fix surface. * distinct silencing mechanism; both share the same fix surface.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner'; import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
// Pass envelope is ['asked', 'plan_ready']; failure signals are // Pass envelope is ['asked', 'plan_ready']; failure signals are
test/skill-e2e-overlay-harness.test.ts
+4 -5
View File
@@ -20,7 +20,8 @@
* Gated by EVALS=1 AND EVALS_TIER=periodic. Never runs under test:gate. * Gated by EVALS=1 AND EVALS_TIER=periodic. Never runs under test:gate.
*/ */
import { describe, test, expect, afterAll } from 'bun:test'; import { test, expect, afterAll } from 'bun:test';
import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
import * as os from 'os'; import * as os from 'os';
@@ -37,11 +38,9 @@ import {
} from './fixtures/overlay-nudges'; } from './fixtures/overlay-nudges';
import { readOverlay } from '../scripts/resolvers/model-overlay'; import { readOverlay } from '../scripts/resolvers/model-overlay';
const evalsEnabled = !!process.env.EVALS; const shouldRun = e2eTierEnabled('periodic');
const periodicTier = process.env.EVALS_TIER === 'periodic';
const shouldRun = evalsEnabled && periodicTier;
const describeE2E = shouldRun ? describe : describe.skip; const describeE2E = describeE2ETier('periodic');
// EvalCollector's tier must be 'e2e' | 'llm-judge' per its type signature. // EvalCollector's tier must be 'e2e' | 'llm-judge' per its type signature.
// The existing paid evals violate this by passing descriptive names like // The existing paid evals violate this by passing descriptive names like
// 'e2e-opus-47' — a pre-existing pattern that only works because bun-test // 'e2e-opus-47' — a pre-existing pattern that only works because bun-test
test/skill-e2e-plan-ceo-finding-count.test.ts
+3 -3
View File
@@ -15,7 +15,8 @@
* test/helpers/claude-pty-runner.ts for runPlanSkillCounting internals. * test/helpers/claude-pty-runner.ts for runPlanSkillCounting internals.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -51,8 +52,7 @@ function pickSkipInterview(fp: AskUserQuestionFingerprint): number {
return 1; return 1;
} }
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N_DISTINCT = 5; const N_DISTINCT = 5;
const FLOOR_DISTINCT = N_DISTINCT - 1; // 4 (D11) const FLOOR_DISTINCT = N_DISTINCT - 1; // 4 (D11)
test/skill-e2e-plan-ceo-finding-floor.test.ts
+3 -3
View File
@@ -4,12 +4,12 @@
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
test( test(
test/skill-e2e-plan-ceo-mode-routing.test.ts
+3 -3
View File
@@ -30,7 +30,8 @@
* SCOPE EXPANSION — "expansion" or "10x" or "delight" or "dream" * SCOPE EXPANSION — "expansion" or "10x" or "delight" or "dream"
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
launchClaudePty, launchClaudePty,
isNumberedOptionListVisible, isNumberedOptionListVisible,
@@ -43,8 +44,7 @@ import {
type ClaudePtySession, type ClaudePtySession,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
interface ModeCase { interface ModeCase {
mode: 'HOLD SCOPE' | 'SCOPE EXPANSION'; mode: 'HOLD SCOPE' | 'SCOPE EXPANSION';
test/skill-e2e-plan-ceo-plan-mode.test.ts
+3 -3
View File
@@ -33,14 +33,14 @@
* See test/helpers/claude-pty-runner.ts for runner internals. * See test/helpers/claude-pty-runner.ts for runner internals.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
assertReportAtBottomIfPlanWritten, assertReportAtBottomIfPlanWritten,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('plan-ceo-review plan-mode smoke (gate)', () => { describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
test('first terminal outcome is asked (Step 0 fires before any plan write)', async () => { test('first terminal outcome is asked (Step 0 fires before any plan write)', async () => {
test/skill-e2e-plan-ceo-review-section-loading.test.ts
+3 -3
View File
@@ -24,15 +24,15 @@
* ~$1-2/run. Periodic tier. * ~$1-2/run. Periodic tier.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
setupSkillDir, setupSkillDir,
skillFromWorktree, skillFromWorktree,
captureSectionReads, captureSectionReads,
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
// Sections every plan-ceo-review run must consult after Step 0. // Sections every plan-ceo-review run must consult after Step 0.
test/skill-e2e-plan-ceo-split-overflow.test.ts
+3 -3
View File
@@ -32,7 +32,8 @@
* Sequential by default. * Sequential by default.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -40,8 +41,7 @@ import {
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
import { FORCING_SPLIT_OVERFLOW_CEO } from './fixtures/forcing-finding-seeds'; import { FORCING_SPLIT_OVERFLOW_CEO } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N = 5; const N = 5;
const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option
test/skill-e2e-plan-design-finding-count.test.ts
+3 -3
View File
@@ -8,7 +8,8 @@
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -16,8 +17,7 @@ import {
assertReviewReportAtBottom, assertReviewReportAtBottom,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N = 5; const N = 5;
const FLOOR = N - 1; const FLOOR = N - 1;
test/skill-e2e-plan-design-finding-floor.test.ts
+3 -3
View File
@@ -4,12 +4,12 @@
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
test( test(
test/skill-e2e-plan-design-plan-mode.test.ts
+3 -3
View File
@@ -9,14 +9,14 @@
* 'plan_ready' are valid pass outcomes. * 'plan_ready' are valid pass outcomes.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
assertReportAtBottomIfPlanWritten, assertReportAtBottomIfPlanWritten,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
// UI-heavy seed with guaranteed design gaps (center-aligned everything, no // UI-heavy seed with guaranteed design gaps (center-aligned everything, no
// empty states, no responsive intent) so the review has real findings to // empty states, no responsive intent) so the review has real findings to
test/skill-e2e-plan-design-with-ui.test.ts
+3 -3
View File
@@ -19,7 +19,8 @@
* contain "no UI scope". * contain "no UI scope".
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as path from 'path'; import * as path from 'path';
import { import {
launchClaudePty, launchClaudePty,
@@ -29,8 +30,7 @@ import {
isPlanReadyVisible, isPlanReadyVisible,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
const ROOT = path.resolve(import.meta.dir, '..'); const ROOT = path.resolve(import.meta.dir, '..');
const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md'); const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
test/skill-e2e-plan-devex-finding-count.test.ts
+3 -3
View File
@@ -8,7 +8,8 @@
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -16,8 +17,7 @@ import {
assertReviewReportAtBottom, assertReviewReportAtBottom,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N = 5; const N = 5;
const FLOOR = N - 1; const FLOOR = N - 1;
test/skill-e2e-plan-devex-finding-floor.test.ts
+3 -3
View File
@@ -4,12 +4,12 @@
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
test( test(
test/skill-e2e-plan-devex-plan-mode.test.ts
+3 -3
View File
@@ -5,15 +5,15 @@
* contract. Exercises the same contract against /plan-devex-review. * contract. Exercises the same contract against /plan-devex-review.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
planFileHasDecisionsSection, planFileHasDecisionsSection,
assertReportAtBottomIfPlanWritten, assertReportAtBottomIfPlanWritten,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('plan-devex-review plan-mode smoke (gate)', () => { describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => {
test/skill-e2e-plan-eng-finding-count.test.ts
+3 -3
View File
@@ -8,7 +8,8 @@
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -16,8 +17,7 @@ import {
assertReviewReportAtBottom, assertReviewReportAtBottom,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N = 5; const N = 5;
const FLOOR = N - 1; // 4 const FLOOR = N - 1; // 4
test/skill-e2e-plan-eng-finding-floor.test.ts
+3 -3
View File
@@ -15,12 +15,12 @@
* Cost: ~$0.50-$1.50 per run depending on early-exit timing. * Cost: ~$0.50-$1.50 per run depending on early-exit timing.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
test( test(
test/skill-e2e-plan-eng-multi-finding-batching.test.ts
+3 -3
View File
@@ -24,7 +24,8 @@
* Tier: periodic (~25 min, ~$5/run). Sequential by default. * Tier: periodic (~25 min, ~$5/run). Sequential by default.
*/ */
import { describe, test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
@@ -32,8 +33,7 @@ import {
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
import { FORCING_BATCHING_ENG } from './fixtures/forcing-finding-seeds'; import { FORCING_BATCHING_ENG } from './fixtures/forcing-finding-seeds';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const N = 4; const N = 4;
const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding
test/skill-e2e-plan-eng-plan-mode.test.ts
+3 -3
View File
@@ -5,15 +5,15 @@
* contract. This file exercises the same contract against /plan-eng-review. * contract. This file exercises the same contract against /plan-eng-review.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
planFileHasDecisionsSection, planFileHasDecisionsSection,
assertReportAtBottomIfPlanWritten, assertReportAtBottomIfPlanWritten,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
// SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the // SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the
// Step 0 complexity check to trigger. Passed via runPlanSkillObservation's // Step 0 complexity check to trigger. Passed via runPlanSkillObservation's
test/skill-e2e-plan-mode-no-op.test.ts
+3 -3
View File
@@ -30,11 +30,11 @@
* change (see 'plan-mode-no-op' in touchfiles.ts). * change (see 'plan-mode-no-op' in touchfiles.ts).
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = describeE2ETier('gate');
const describeE2E = shouldRun ? describe : describe.skip;
const PLAN_MODE_REMINDER = const PLAN_MODE_REMINDER =
'Plan mode is active. The user indicated that they do not want you to execute yet'; 'Plan mode is active. The user indicated that they do not want you to execute yet';
test/skill-e2e-setup-gbrain-bad-token.test.ts
+3 -3
View File
@@ -8,7 +8,8 @@
// //
// Cost: ~$0.30-$0.50 per run. Gate-tier (EVALS=1 EVALS_TIER=gate). // Cost: ~$0.30-$0.50 per run. Gate-tier (EVALS=1 EVALS_TIER=gate).
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
@@ -17,8 +18,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f
// Periodic-tier (companion to skill-e2e-setup-gbrain-remote.test.ts). // Periodic-tier (companion to skill-e2e-setup-gbrain-remote.test.ts).
// Deterministic gate coverage lives in setup-gbrain-path4-structure.test.ts. // Deterministic gate coverage lives in setup-gbrain-path4-structure.test.ts.
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
function startStub401(): Promise<{ url: string; close: () => Promise<void> }> { function startStub401(): Promise<{ url: string; close: () => Promise<void> }> {
return new Promise((resolve) => { return new Promise((resolve) => {
test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts
+3 -3
View File
@@ -18,7 +18,8 @@
// //
// Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic). // Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic).
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
@@ -29,8 +30,7 @@ import {
resolveClaudeBinary, resolveClaudeBinary,
} from './helpers/agent-sdk-runner'; } from './helpers/agent-sdk-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
/** /**
* Minimal stub MCP server that returns success on initialize / tools/list. * Minimal stub MCP server that returns success on initialize / tools/list.
test/skill-e2e-setup-gbrain-remote.test.ts
+3 -3
View File
@@ -11,7 +11,8 @@
// //
// See setup-gbrain/SKILL.md.tmpl Step 4 (Path 4) for the contract under test. // See setup-gbrain/SKILL.md.tmpl Step 4 (Path 4) for the contract under test.
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
@@ -22,8 +23,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f
// non-deterministic (it sometimes skips Step 8 CLAUDE.md write, sometimes // non-deterministic (it sometimes skips Step 8 CLAUDE.md write, sometimes
// shortcuts past the verify helper). The deterministic gate coverage for // shortcuts past the verify helper). The deterministic gate coverage for
// Path 4 lives in test/setup-gbrain-path4-structure.test.ts (free, <200ms). // Path 4 lives in test/setup-gbrain-path4-structure.test.ts (free, <200ms).
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
// Spin up a stub MCP server that responds to initialize + tools/list. // Spin up a stub MCP server that responds to initialize + tools/list.
function startStubMcpServer(opts: { failWithStatus?: number; failBody?: string } = {}): Promise<{ url: string; close: () => Promise<void> }> { function startStubMcpServer(opts: { failWithStatus?: number; failBody?: string } = {}): Promise<{ url: string; close: () => Promise<void> }> {
test/skill-e2e-ship-idempotency.test.ts
+3 -3
View File
@@ -30,7 +30,8 @@
* Cost: ~$2-4/run. Periodic tier — long, runs weekly. * Cost: ~$2-4/run. Periodic tier — long, runs weekly.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
@@ -41,8 +42,7 @@ import {
isNumberedOptionListVisible, isNumberedOptionListVisible,
} from './helpers/claude-pty-runner'; } from './helpers/claude-pty-runner';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
interface ShipFixture { interface ShipFixture {
workTree: string; workTree: string;
test/skill-e2e-ship-section-loading.test.ts
+3 -3
View File
@@ -23,15 +23,15 @@
* Periodic tier. * Periodic tier.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import { import {
setupSkillDir, setupSkillDir,
skillFromWorktree, skillFromWorktree,
captureSectionReads, captureSectionReads,
} from './helpers/auq-sdk-capture'; } from './helpers/auq-sdk-capture';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const runId = `ship-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; const runId = `ship-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
// Sections every version-changing ship must consult. // Sections every version-changing ship must consult.
test/skill-e2e-spec-execute.test.ts
+3 -3
View File
@@ -16,12 +16,12 @@
* minimum smoke that proves --execute end-to-end works. * minimum smoke that proves --execute end-to-end works.
*/ */
import { describe, test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = describeE2ETier('periodic');
const describeE2E = shouldRun ? describe : describe.skip;
const ROOT = path.resolve(import.meta.dir, '..'); const ROOT = path.resolve(import.meta.dir, '..');