From 2ec4dcf1050dce9ecba6124ba48dc2f441099a26 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 05:06:57 +0000 Subject: [PATCH] fix(evals): every E2E key's dep list names its own declaring test file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 129-of-177 keys omitted their own test file, so editing only a test's prompt or assertions selected NOTHING — the changed test never ran on the change that changed it. 135 keys self-registered (110 E2E + 25 LLM-judge), resolved by strict declaration evidence (testName:/ testIfSelected/judge call sites), with skill-name false positives excluded. e2e-tier-alignment's warn-only branch for unregistered files is now a hard failure with a 4-entry KNOWN_UNREGISTERED ratchet (template- literal testNames, fail-open-safe) + a burn-down test so the set only shrinks. Selection sanity: a one-file diff on skill-e2e-qa-workflow now selects its 4 tests (was 0); skill-llm-eval 0 → 25. Known follow-ups (filed): 15 E2E + 2 judge PHANTOM keys select tests that exist nowhere; codex-e2e-plan-format's testIfSelected names have no map keys (run-all only). Co-Authored-By: Claude Fable 5 --- test/e2e-tier-alignment.test.ts | 77 ++++++++-- test/helpers/touchfiles-data.ts | 265 ++++++++++++++++---------------- 2 files changed, 203 insertions(+), 139 deletions(-) diff --git a/test/e2e-tier-alignment.test.ts b/test/e2e-tier-alignment.test.ts index 92d937e50..ba6bcee10 100644 --- a/test/e2e-tier-alignment.test.ts +++ b/test/e2e-tier-alignment.test.ts @@ -11,9 +11,15 @@ * Mapping rule (test filenames do NOT map mechanically to tier keys): for each * `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the * E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If - * found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files - * not named in any dep list are REPORTED as unmapped (a nudge to add them to - * their eval's dep list), never silently skipped. + * found under key K, the file's self-gate tier must equal E2E_TIERS[K]. + * + * Self-registration is a HARD invariant (the dep-list sweep): every + * skill-e2e file must be named in at least one touchfiles dep list, so that + * editing only the test's prompt/assertions diff-selects the test itself. + * Before the sweep, 129 of ~177 E2E keys did not list their own declaring + * file — a changed test never re-ran on its own change. Files that genuinely + * cannot be mapped (no E2E map key exists for them) sit in KNOWN_UNREGISTERED + * below; that set is a ratchet, it only shrinks. */ import { describe, test, expect } from 'bun:test'; @@ -35,6 +41,26 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g; // the declared tier exactly like the raw predicate's tier literal did. const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g; +/** + * Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in + * test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO + * touchfiles dep list because no E2E map key exists for them. Every entry + * carries a one-line reason. Do NOT add new files here — give the test an + * E2E map key (touchfiles + tier) and register the file in its dep list. + * A stale entry (file deleted, or file now registered) FAILS the suite — + * delete it. Target: empty set. + */ +const KNOWN_UNREGISTERED = new Set([ + // Standalone periodic self-gated probe; template-literal testNames (auq-consistency-${i}), no E2E map key — fail-open-safe, runs on every periodic sweep. + 'test/skill-e2e-auq-consistency.test.ts', + // Standalone periodic self-gated matrix; template-literal testNames (auq-matrix-${m.skill}), no E2E map key — fail-open-safe, runs on every periodic sweep. + 'test/skill-e2e-auq-matrix.test.ts', + // Standalone periodic self-gated A/B probe; template-literal testNames (auq-ab-${label}), no E2E map key — fail-open-safe, runs on every periodic sweep. + 'test/skill-e2e-auq-verbose-vs-carved-ab.test.ts', + // bin-script pipeline test (spawns bun scripts, no model spend) that lives under the skill-e2e-* glob; no E2E map key exists for it. + 'test/skill-e2e-memory-pipeline.test.ts', +]); + describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => { const testFiles = readdirSync(TEST_DIR) .filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts')) @@ -44,14 +70,28 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => test('every self-gated test file named in a dep list matches its declared tier', () => { const misaligned: string[] = []; + const unregistered: string[] = []; const reported: string[] = []; for (const file of testFiles) { const content = readFileSync(path.join(TEST_DIR, file), 'utf-8'); + const repoPath = `test/${file}`; + + // HARD self-registration invariant, independent of self-gate shape: + // a skill-e2e file named in no dep list means editing the test itself + // selects nothing — the changed test never re-runs on its own change. + const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath)); + if (owningKeys.length === 0 && !KNOWN_UNREGISTERED.has(repoPath)) { + unregistered.push( + `${repoPath}: not named in any touchfiles dep list — editing this test file would ` + + 'never diff-select it. Add the file path to its E2E map key\'s dep list in ' + + 'test/helpers/touchfiles-data.ts (do NOT extend KNOWN_UNREGISTERED for new files).', + ); + } + const tiers = new Set(); for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]); for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]); - const repoPath = `test/${file}`; if (tiers.size === 0) { // Every skill-e2e file is expected to self-gate; zero matches means // either a genuinely ungated file or a gate shape the regex can't @@ -65,8 +105,9 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => } const selfTier = [...tiers][0]; - const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath)); if (owningKeys.length === 0) { + // Only KNOWN_UNREGISTERED files reach here (anything else already + // hard-failed above) — keep the visible nudge. reported.push(`${repoPath} (self-gates '${selfTier}'): not named in any touchfiles dep list`); continue; } @@ -86,18 +127,36 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => } } - // Reported, not asserted: coverage holes the invariant can see but not - // arbitrate. Add the test file to its eval's dep list (or a tier entry - // for the key) to bring it under the invariant. + // Reported, not asserted: tier-observability holes the invariant can see + // but not arbitrate (map-driven files legitimately have no whole-file + // self-gate; ratcheted files stay visible). Registration itself is + // asserted below. if (reported.length > 0) { console.warn( - `[tier-alignment] ${reported.length} file(s) outside the invariant:\n ` + reported.join('\n '), + `[tier-alignment] ${reported.length} file(s) outside the tier invariant:\n ` + reported.join('\n '), ); } + expect(unregistered).toEqual([]); expect(misaligned).toEqual([]); }); + // Ratchet cleanup enforcement (same contract as evals-workflow-matrix's + // burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or + // whose file is now named in a dep list, is stale — delete the entry so + // the set can only shrink. + test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => { + const stale = [...KNOWN_UNREGISTERED].filter((repoPath) => { + const file = repoPath.replace(/^test\//, ''); + if (!testFiles.includes(file)) return true; // file gone + return Object.keys(allDeps).some((k) => allDeps[k].includes(repoPath)); // now registered + }); + expect( + stale, + 'Entry registered in a dep list or file removed — delete it from KNOWN_UNREGISTERED.', + ).toEqual([]); + }); + // HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when // none of the file's MAPPED test names (E2E map keys quoted in its source, // union E2E map keys whose dep list registers the file) are diff-selected. diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 90b8877a4..b6a6f4645 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -22,8 +22,8 @@ */ export const E2E_TOUCHFILES: Record = { // Browse core (+ test-server dependency) - 'browse-basic': ['browse/src/**', 'browse/test/test-server.ts'], - 'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts'], + 'browse-basic': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'], + 'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'], // Hermetic isolation canaries (hermetic-env.ts is also a GLOBAL touchfile; // these entries exist so the canaries themselves stay tier-classified) @@ -37,21 +37,21 @@ export const E2E_TOUCHFILES: Record = { 'first-task-scaffold': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'], // SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs) - 'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], + 'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], + 'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], + 'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], - 'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'], + 'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], + 'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log', 'test/skill-e2e-bws.test.ts'], // QA (+ test-server dependency) - 'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'], - 'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'], - 'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'], - 'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'], - 'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'], - 'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'], - 'qa-bootstrap': ['qa/**', 'ship/**'], + 'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'], + 'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'], + 'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'], + 'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'], + 'qa-only-no-fix': ['qa-only/**', 'qa/templates/**', 'test/skill-e2e-qa-workflow.test.ts'], + 'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'], + 'qa-bootstrap': ['qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts'], // Review 'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts'], @@ -60,27 +60,27 @@ export const E2E_TOUCHFILES: Record = { 'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-review.test.ts'], // Review Army (specialist dispatch) - 'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'], - 'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'], - 'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts'], - 'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'], - 'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'], - 'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'], - 'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'], + 'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'], + 'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'], + 'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'], + 'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'], + 'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'], + 'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'], + 'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'], // Office Hours - 'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts'], - 'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'], - 'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'], + 'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], + 'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'], + 'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'], // Plan reviews - 'plan-ceo-review': ['plan-ceo-review/**'], - 'plan-ceo-review-selective': ['plan-ceo-review/**'], - 'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'], - 'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'], - 'plan-eng-review': ['plan-eng-review/**'], - 'plan-eng-review-artifact': ['plan-eng-review/**'], - 'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'], + 'plan-ceo-review': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'], + 'plan-ceo-review-selective': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'], + 'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], + 'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts'], + 'plan-eng-review': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'], + 'plan-eng-review-artifact': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'], + 'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], // Plan-mode smoke tests — gate-tier safety regression tests. Each test file // contains TWO test cases as of v1.21: the baseline plan-mode case and the @@ -91,7 +91,7 @@ export const E2E_TOUCHFILES: Record = { // regression test outcome between 'asked' and 'auto_decided'. 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], - 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'], + 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; @@ -117,7 +117,7 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts'], + 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], // Conductor → prose decision brief (Conductor signal makes prose the default; // the PreToolUse hook denies the flaky tool). Touches the resolver that owns @@ -128,18 +128,18 @@ export const E2E_TOUCHFILES: Record = { // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'preamble-script-ab': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/skill-e2e-preamble-script-ab.test.ts'], - 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'], + 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'], 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], 'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'], 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'], 'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'], - 'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], + 'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-plan-ceo-review-section-loading.test.ts'], // Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design, // devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS; // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. - 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], + 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'], 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], 'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'], @@ -185,18 +185,18 @@ export const E2E_TOUCHFILES: Record = { // AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10) // Fires when either template OR the two preamble resolvers change. - 'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'], - 'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'], - 'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'], - 'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'], + 'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'], + 'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'], + 'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'], + 'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'], // v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals. // Dependencies: same as format-mode + the 4 plan-review templates + overlay. // All periodic-tier (non-deterministic Opus 4.7 behavior). - 'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], - 'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], + 'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'], + 'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'], + 'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'], + 'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'], // Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble 'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], @@ -208,24 +208,24 @@ export const E2E_TOUCHFILES: Record = { 'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'], // /plan-tune (v1 observational) - 'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'], + 'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile', 'test/skill-e2e-plan-tune.test.ts'], // /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12) - 'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**'], - 'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts'], - 'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts'], - 'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md'], - 'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**'], + 'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'], + 'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'], + 'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'], + 'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md', 'test/skill-e2e-plan-tune-cathedral.test.ts'], + 'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'], // Codex offering verification - 'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts'], - 'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'], - 'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'], - 'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'], + 'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], + 'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], + 'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], + 'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'], // Ship 'ship-base-branch': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-review-attribution.test.ts'], - 'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts'], + 'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-workflow.test.ts'], 'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'], // Retro @@ -236,51 +236,51 @@ export const E2E_TOUCHFILES: Record = { 'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'], // CSO - 'cso-full-audit': ['cso/**'], - 'cso-diff-mode': ['cso/**'], - 'cso-infra-scope': ['cso/**'], + 'cso-full-audit': ['cso/**', 'test/skill-e2e-cso.test.ts'], + 'cso-diff-mode': ['cso/**', 'test/skill-e2e-cso.test.ts'], + 'cso-infra-scope': ['cso/**', 'test/skill-e2e-cso.test.ts'], // Learnings - 'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts'], + 'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts', 'test/skill-e2e-learnings.test.ts'], // Session Intelligence (timeline, context recovery, /context-save + /context-restore) - 'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read'], - 'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**'], - 'context-save-writes-file': ['context-save/**', 'bin/gstack-slug'], - 'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug'], + 'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read', 'test/skill-e2e-session-intelligence.test.ts'], + 'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**', 'test/skill-e2e-session-intelligence.test.ts'], + 'context-save-writes-file': ['context-save/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'], + 'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'], // Context skills E2E (live-fire, Skill-tool routing path) — see // test/skill-e2e-context-skills.test.ts. These are periodic-tier because // each one spawns claude -p and costs ~$0.20-$0.40. Collectively they // verify the thing the /checkpoint → /context-save rename was for. - 'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts'], - 'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug'], - 'context-restore-fragment-match': ['context-restore/**'], - 'context-restore-empty-state': ['context-restore/**'], - 'context-restore-list-delegates': ['context-restore/**'], - 'context-restore-legacy-compat': ['context-restore/**'], - 'context-save-list-current-branch': ['context-save/**'], - 'context-save-list-all-branches': ['context-save/**'], + 'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts'], + 'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-context-skills.test.ts'], + 'context-restore-fragment-match': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-restore-empty-state': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-restore-list-delegates': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-restore-legacy-compat': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-save-list-current-branch': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-save-list-all-branches': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'], // Document-release - 'document-release': ['document-release/**'], + 'document-release': ['document-release/**', 'test/skill-e2e-workflow.test.ts'], // Codex (Claude E2E — tests /codex skill via Claude) - 'codex-review': ['codex/**'], + 'codex-review': ['codex/**', 'test/skill-e2e-workflow.test.ts'], // Codex E2E (tests skills via Codex CLI + worktree) - 'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'], - 'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'], + 'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], + 'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'], // GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill) 'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'], // Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks) - 'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'], + 'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'], // Coverage audit (shared fixture) + triage + gates - 'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode'], + 'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode', 'test/skill-e2e-workflow.test.ts'], 'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'], 'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'], 'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'], @@ -292,12 +292,12 @@ export const E2E_TOUCHFILES: Record = { 'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'], // Design - 'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts'], - 'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts'], - 'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts'], - 'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts'], - 'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'], - 'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts'], + 'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'], + 'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], + 'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], + 'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], + 'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], + 'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'], // Design Shotgun 'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'], @@ -306,24 +306,24 @@ export const E2E_TOUCHFILES: Record = { // /diagram (diagram-render bundle consumers). Triplet = deterministic // functional (gate); authoring quality = LLM-judged benchmark (periodic). - 'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts'], - 'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts'], + 'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts', 'test/skill-e2e-diagram.test.ts'], + 'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-diagram.test.ts'], // gstack-upgrade - 'gstack-upgrade-happy-path': ['gstack-upgrade/**'], + 'gstack-upgrade-happy-path': ['gstack-upgrade/**', 'test/skill-e2e-workflow.test.ts'], // Deploy skills - 'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts'], - 'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug'], - 'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read'], - 'canary-workflow': ['canary/**', 'browse/src/**'], - 'benchmark-workflow': ['benchmark/**', 'browse/src/**'], - 'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts'], + 'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'], + 'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug', 'test/skill-e2e-deploy.test.ts'], + 'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read', 'test/skill-e2e-deploy.test.ts'], + 'canary-workflow': ['canary/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'], + 'benchmark-workflow': ['benchmark/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'], + 'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'], // Autoplan 'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'], - 'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'], + 'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts'], // Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs 'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'], @@ -336,41 +336,46 @@ export const E2E_TOUCHFILES: Record = { 'scrape-match-path': [ 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', 'browser-skills/hackernews-frontpage/**', + 'test/skill-e2e-skillify.test.ts', ], 'scrape-prototype-path': [ 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', + 'test/skill-e2e-skillify.test.ts', ], 'skillify-happy-path': [ 'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', + 'test/skill-e2e-skillify.test.ts', ], 'skillify-provenance-refusal': [ 'skillify/**', 'browse/src/browser-skill-write.ts', + 'test/skill-e2e-skillify.test.ts', ], 'skillify-approval-reject': [ 'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts', + 'test/skill-e2e-skillify.test.ts', ], // Skill routing — journey-stage tests (depend on ALL skill descriptions) - 'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], - 'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], + 'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], + 'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'], // Opus 4.7 behavior evals — keys match testName: values in the test file. // Routing sub-tests use template literal `routing-${c.name}` testNames, // which the touchfile completeness scanner skips; they inherit selection // from the file-level touchfile entry via GLOBAL_TOUCHFILES. 'fanout-arm-overlay-on': - ['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'], + ['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'], 'fanout-arm-overlay-off': - ['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'], + ['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'], // Overlay efficacy harness (SDK) — measures whether overlay nudges change // behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code @@ -754,49 +759,49 @@ export const E2E_TIERS: Record = { * LLM-judge test touchfiles — keyed by test description string. */ export const LLM_JUDGE_TOUCHFILES: Record = { - 'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'], - 'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'], - 'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'], - 'setup block': ['SKILL.md', 'SKILL.md.tmpl'], - 'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'], - 'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'], - 'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'], - 'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'], - 'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'], - 'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'], + 'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/skill-llm-eval.test.ts'], + 'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts', 'test/skill-llm-eval.test.ts'], + 'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'test/skill-llm-eval.test.ts'], + 'setup block': ['SKILL.md', 'SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'], + 'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'], // Ship & Release - 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'], - 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'], + 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Plan Reviews - 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'], - 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'], + 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // /spec authored-spec quality (paid LLM-judge — periodic-tier). - 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'], + 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Design skills - 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'], - 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'], + 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Office Hours 'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], 'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], // Deploy skills - 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**'], - 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl'], - 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl'], - 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'], + 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'], + 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Other skills - 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'], - 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'], - 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'], + 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], // Voice directive - 'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'], + 'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'], }; /**