From 10b6cf5b8033e4d1ceb33a3cf3793331963dd809 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 8 Sep 2026 18:14:04 +0000 Subject: [PATCH] test: isolate installed runtime paths in planning evals --- test/autoplan-setup-question.test.ts | 36 ++++++ test/helpers/autoplan-setup-question.ts | 4 +- test/helpers/claude-pty-runner.ts | 10 +- test/helpers/hermetic-env.ts | 11 +- test/helpers/hermetic-skill-runtime.ts | 66 +++++++++++ test/helpers/touchfiles-data.ts | 44 +++---- test/hermetic-skill-runtime.test.ts | 147 ++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 30 deletions(-) create mode 100644 test/helpers/hermetic-skill-runtime.ts create mode 100644 test/hermetic-skill-runtime.test.ts diff --git a/test/autoplan-setup-question.test.ts b/test/autoplan-setup-question.test.ts index 29068de42..def47dd86 100644 --- a/test/autoplan-setup-question.test.ts +++ b/test/autoplan-setup-question.test.ts @@ -87,6 +87,42 @@ const FRESH_RETRY_CAPTURE = [ ].join('\n'); describe('autoplan routing setup handling', () => { + // Source-F retry's first complete frame preceded damaged terminal redraws. + const F_SETUP_CAPTURE = [ + 'Planning: /tmp/hermetic/.claude/plans/deep-coalescing-valiant.md', + '☐Skillrouting', + "│gstackworksbestwhenyourproject'sCLAUDE.mdincludesskillroutingrules.Addthemnow?", + '❯1.AddroutingrulestoCLAUDE.md', + 'AppendsskillroutingrulestoCLAUDE.mdsogstackauto-invokestherightskillforcommonrequests(review,ship,', + 'investigate,etc.).Willbecommittedtotherepo.(recommended)', + '2.Nothanks,skip', + "I'llinvokeskillsmanually.Youcanaddroutinglater.", + '3.Typesomething.', + '4.Chataboutthis', + 'Entertoselect·↑/↓tonavigate·Esctocancel', + ].join('\r\r'); + + test('answers the captured combined decline action once, regardless of option order', () => { + const seen = new Set(); + expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE, seen)).toBe('1\r'); + expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE, seen)).toBeNull(); + const reordered = F_SETUP_CAPTURE.replace('❯1.AddroutingrulestoCLAUDE.md', '❯1.Nothanks,skip') + .replace('2.Nothanks,skip', '2.AddroutingrulestoCLAUDE.md'); + expect(autoplanRoutingSetupInput(reordered, new Set())).toBe('2\r'); + expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip—invoke skills manually'), new Set())).toBe('1\r'); + }); + + test('does not infer a routing answer from damaged, ambiguous, or unrelated setup choices', () => { + for (const frame of [ + F_SETUP_CAPTURE.replace('Addroutingrules', 'Addrutingrules'), + F_SETUP_CAPTURE.replace('Nothanks,skip', 'Nothank,skip'), + F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip the review'), + F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip then delete CLAUDE.md'), + F_SETUP_CAPTURE.replace('3.Typesomething.', '3.Skip'), + F_SETUP_CAPTURE.replace("gstackworksbestwhenyourproject'sCLAUDE.mdincludesskillroutingrules.Addthemnow?", 'Which routing design should the application use?'), + ]) expect(autoplanRoutingSetupInput(frame, new Set()), frame).toBeNull(); + }); + test('answers the fresh retry manual-invocation setup once in either option order', () => { const seen = new Set(); expect(autoplanRoutingSetupInput(FRESH_RETRY_CAPTURE, seen)).toBe('1\r'); diff --git a/test/helpers/autoplan-setup-question.ts b/test/helpers/autoplan-setup-question.ts index a9af8a3ad..0f67b1c0b 100644 --- a/test/helpers/autoplan-setup-question.ts +++ b/test/helpers/autoplan-setup-question.ts @@ -24,7 +24,9 @@ export function autoplanRoutingSetupInput(visible: string, seen: Set): s const title = option.title.replace(/\(Recommended\)$/i, ''); const prefix = /^(?:Nothanks|Skip)(?:[,—–-])?/i.exec(title); if (!prefix) return false; - const action = title.slice(prefix[0].length); + // 'No thanks' can be followed by the same explicit Skip action. Strip + // that decline verb before checking any optional manual-invocation text. + const action = title.slice(prefix[0].length).replace(/^skip(?:[,—–-])?/i, ''); return action === '' || /^(?:manual(?:invocation)?|(?:I['’]ll)?invoke(?:skills)?manually)$/i.test(action); }); if (add.length !== 1 || decline.length !== 1 || add[0]!.index === decline[0]!.index) return null; diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 0dab56b4e..45c0e3e24 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -27,6 +27,7 @@ import * as os from 'os'; import * as path from 'path'; import { stripVTControlCharacters } from 'node:util'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env'; +import { withHermeticSkillRuntime } from './hermetic-skill-runtime'; import { createPlanCountFixture } from './plan-count-fixture'; import { createPlanCountSnapshotWriter } from './plan-count-artifacts'; import { readPlanCountTranscript, unresolvedPlanQuestionCalls, type NativePlanQuestionCall, type PlanCountTranscript } from './plan-count-transcript'; @@ -1600,9 +1601,16 @@ export async function launchClaudePty( // Hermetic by default (test/helpers/hermetic-env.ts): operator session // context never reaches the child; per-test opts.env merges last. - const childEnv = hermeticChildEnv(opts.env); + let childEnv = hermeticChildEnv(opts.env); if (opts.seedSkills && hermetic && !opts.env?.CLAUDE_CONFIG_DIR) { childEnv.CLAUDE_CONFIG_DIR = hermeticSkillsConfigDir(); + if (opts.env?.HOME === undefined) { + const runtime = withHermeticSkillRuntime(childEnv); + childEnv = runtime.env; + // Lazy sections are installed runtime inputs outside the fixture cwd. + // Grant this known directory, preserving all other permission decisions. + args.push('--add-dir', runtime.root); + } } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts index 94c8588cc..006c48a09 100644 --- a/test/helpers/hermetic-env.ts +++ b/test/helpers/hermetic-env.ts @@ -268,12 +268,11 @@ let cachedSkillsConfigDir: string | null = null; * cover it. Ends in `/.claude` for the same plan-path anchoring reason as * HermeticDirs.configDir. * - * Two intentional non-hermetic edges: - * - Seeding reads the LIVE repo tree BY DESIGN — the skills ARE the subject - * under test; a snapshot would measure stale copies. - * - HOME is not hermeticized, so the ~64 absolute - * `~/.claude/skills/gstack/...` preamble references inside each SKILL.md - * still resolve to the operator install (same limitation as CI). + * Seeding reads the live repo tree: the skills are the subject under test. + * For default seeded PTY sessions, launchClaudePty also supplies an owned HOME + * via hermetic-skill-runtime so literal runtime and lazy-section paths reach + * this same checkout. Explicit HOME/config overrides remain caller-owned; + * this registration helper itself does not change their environment. */ export function hermeticSkillsConfigDir(): string { if (cachedSkillsConfigDir) return cachedSkillsConfigDir; diff --git a/test/helpers/hermetic-skill-runtime.ts b/test/helpers/hermetic-skill-runtime.ts new file mode 100644 index 000000000..90f4ee1e4 --- /dev/null +++ b/test/helpers/hermetic-skill-runtime.ts @@ -0,0 +1,66 @@ +/** + * Runtime paths for PTY sessions that seed the checkout's Claude skills. + * + * Claude's registry uses CLAUDE_CONFIG_DIR, but generated skills also read + * ~/.claude/skills/gstack and execute $HOME/.claude/skills/gstack/bin tools. + * Point those unchanged paths at the same checkout, never an operator install. + * This HOME is owned by the existing hermetic runRoot and its exit/GC cleanup. + * + * Two intentional shared resources survive the HOME change: nested Codex's + * configured auth/model directory, and Playwright's existing browser cache. + * Claude auth stays in the existing seeded config/API env; gstack state stays + * at the caller's GSTACK_HOME. No operator skill, hooks, or settings are copied. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { getHermeticDirs } from './hermetic-env'; + +let cachedRuntime: { home: string; root: string } | undefined; + +export function hermeticSkillRuntime(): { home: string; root: string } { + if (cachedRuntime) return cachedRuntime; + const home = fs.mkdtempSync(path.join(getHermeticDirs().runRoot, 'skill-home-')); + const root = path.join(home, '.claude', 'skills', 'gstack'); + try { + fs.mkdirSync(path.dirname(root), { recursive: true }); + fs.symlinkSync(path.resolve(import.meta.dir, '..', '..'), root, 'dir'); + } catch (error) { + fs.rmSync(home, { recursive: true, force: true }); + throw error; + } + cachedRuntime = { home, root }; + return cachedRuntime; +} + +/** Keep the cache the child used before HOME isolation (Playwright defaults). */ +function browserCache(env: Record, originalHome: string): string { + if (env.PLAYWRIGHT_BROWSERS_PATH) return env.PLAYWRIGHT_BROWSERS_PATH; + const base = process.platform === 'darwin' + ? path.join(originalHome, 'Library', 'Caches') + : process.platform === 'win32' + ? env.LOCALAPPDATA || path.join(originalHome, 'AppData', 'Local') + : env.XDG_CACHE_HOME || path.join(originalHome, '.cache'); + return path.join(base, 'ms-playwright'); +} + +export function withHermeticSkillRuntime( + env: Record, + operatorEnv: NodeJS.ProcessEnv = process.env, +): { env: Record; root: string } { + const originalHome = env.HOME || os.homedir(); + const runtime = hermeticSkillRuntime(); + // An explicit empty per-test value resets Codex to its original HOME default. + const codexHome = env.CODEX_HOME !== undefined + ? env.CODEX_HOME || path.join(originalHome, '.codex') + : operatorEnv.CODEX_HOME || path.join(originalHome, '.codex'); + return { + root: runtime.root, + env: { + ...env, + HOME: runtime.home, + CODEX_HOME: codexHome, + PLAYWRIGHT_BROWSERS_PATH: browserCache(env, originalHome), + }, + }; +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index efc9eeaea..56055ea56 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -100,17 +100,17 @@ export const E2E_TOUCHFILES: Record = { // include question-tuning.ts and generate-ask-user-format.ts because the // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. - 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], - 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], - 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], - 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], + 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], + 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], + 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], + 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; // in CI these run CONCURRENT with the rest of the pty-plan-smoke suite // (--max-concurrency + --retry 1), so worst-case cost is ~2x a single // pass of each, sharing the API budget with sibling tests — not the // sequential ~+10min a local read suggests. - 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], + 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], // v1.21+ AskUserQuestion-blocked regression tests — Conductor launches // claude with `--disallowedTools AskUserQuestion --permission-mode default` @@ -120,7 +120,7 @@ export const E2E_TOUCHFILES: Record = { // INSIDE the existing 4 plan-X-review-plan-mode test files (covered // transitively by the entries above). Two new standalone files exist for // skills with no prior plan-mode test: - 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], + 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], 'office-hours-phase4-fork': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'], 'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'], // v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution @@ -128,20 +128,20 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], + 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], // Conductor → prose decision brief (Conductor signal makes prose the default; // the PreToolUse hook denies the flaky tool). Touches the resolver that owns // the Conductor rule, the preamble signal, the hook, and the detection helper. - 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'], + 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'], // Real-PTY E2E batch (#6 new tests on the harness). // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'], - 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], - 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], - 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'], + 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], + 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], + 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'], 'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], @@ -154,34 +154,34 @@ export const E2E_TOUCHFILES: Record = { // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts'], - 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'], + 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], - 'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], - 'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], - 'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], + 'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], + 'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], + 'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], + 'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], // Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript // bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any // review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal // "did agent fire ANY AUQ?" observer that exits early on first non-permission // numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total. - 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], - 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], - 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], - 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], + 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], + 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], + 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], + 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], // Multi-finding batching regression — periodic tier complement to the // gate-tier finding-floor. Catches the May 2026 transcript shape where // a model fires one AUQ then batches the rest into a "## Decisions to // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. - 'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], - 'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], + 'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], + 'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via diff --git a/test/hermetic-skill-runtime.test.ts b/test/hermetic-skill-runtime.test.ts new file mode 100644 index 000000000..38daa989d --- /dev/null +++ b/test/hermetic-skill-runtime.test.ts @@ -0,0 +1,147 @@ +/** Free installed-runtime probe through the real PTY launcher; no provider calls. */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createHash } from 'node:crypto'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; +const ROOT = path.resolve(import.meta.dir, '..'); +const digest = (file: string) => createHash('sha256').update(fs.readFileSync(file)).digest('hex'); + +describe('hermetic seeded PTY runtime', () => { + test('runtime helper and regression select every PTY consumer', () => { + const consumers = Object.entries(E2E_TOUCHFILES).filter(([, files]) => files.includes('test/helpers/claude-pty-runner.ts')).map(([name]) => name).sort(); + expect(consumers.length).toBeGreaterThan(15); + for (const file of ['test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts']) + expect(selectTests([file], E2E_TOUCHFILES).selected.sort()).toEqual(consumers); + }); + test.skipIf(process.platform === 'win32')('uses current lazy files and tools with scoped access, preserving auth, caches, and explicit overrides', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-runtime-probe-')); + const operatorHome = path.join(dir, 'operator-home'); + const oldRoot = path.join(operatorHome, '.claude', 'skills', 'gstack'); + const section = 'autoplan/sections/ceo-phase.md'; + fs.mkdirSync(path.join(oldRoot, 'autoplan', 'sections'), { recursive:true }); + fs.mkdirSync(path.join(oldRoot, 'bin')); + fs.writeFileSync(path.join(oldRoot, section), 'STALE OPERATOR PHASE\n'); + fs.writeFileSync(path.join(oldRoot, 'bin/gstack-config'), '#!/bin/sh\necho stale-runtime\n', { mode:0o755 }); + const state = path.join(dir, 'state'); + fs.mkdirSync(state); + fs.writeFileSync(path.join(state, 'config.yaml'), 'codex_reviews: disabled\n'); + const codexDefault = path.join(operatorHome, '.codex'); + const codexConfigured = path.join(dir, 'configured-codex'); + const codexOverride = path.join(dir, 'override-codex'); + for (const home of [codexDefault,codexConfigured,codexOverride]) { + fs.mkdirSync(home); + fs.writeFileSync(path.join(home,'config.toml'), 'model = "unchanged-fixture-model"\n'); + fs.writeFileSync(path.join(home,'auth.json'), '{"fixture_auth":true}\n'); + } + const customConfig = path.join(dir,'custom-config'); + fs.mkdirSync(customConfig); + const fake = path.join(dir,'fake-claude'); + fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw` +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +const root = path.join(process.env.HOME,'.claude','skills','gstack'); +const args = process.argv.slice(2); +const addDir = args.includes('--add-dir') ? args[args.indexOf('--add-dir')+1] : null; +const phase = spawnSync('bash',['-c','cat ~/.claude/skills/gstack/autoplan/sections/ceo-phase.md'],{timeout:5000}); +const config = spawnSync('bash',['-c','"$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews'],{timeout:5000}); +const codexHome = process.env.CODEX_HOME || path.join(process.env.HOME,'.codex'); +const record = { + pid:process.pid, home:process.env.HOME, root:fs.realpathSync(root), args, + phaseExit:phase.status, phaseHash:createHash('sha256').update(phase.stdout).digest('hex'), + configExit:config.status, configValue:config.stdout.toString().trim(), + configDir:process.env.CLAUDE_CONFIG_DIR, state:process.env.GSTACK_HOME, + runtimeAllowed:addDir !== null && fs.realpathSync(addDir) === fs.realpathSync(root), + claudeAuthPreserved:process.env.ANTHROPIC_API_KEY === 'fixture-api-key', + codexHome, codexConfig:fs.readFileSync(path.join(codexHome,'config.toml'),'utf8'), + codexAuthPreserved:JSON.parse(fs.readFileSync(path.join(codexHome,'auth.json'),'utf8')).fixture_auth === true, + browserCache:process.env.PLAYWRIGHT_BROWSERS_PATH, +}; +fs.writeFileSync(process.env.RUNTIME_RECORD,JSON.stringify(record)); +if (process.env.RUNTIME_FAIL === '1') throw new Error('fixture startup failure'); +process.stdout.write('RUNTIME_PROBE_READY\n'); +process.stdin.resume(); +`); + fs.chmodSync(fake,0o755); + const runner = pathToFileURL(path.join(ROOT,'test/helpers/claude-pty-runner.ts')).href; + const cases = [ + {name:'seeded-default',seed:true,owned:true}, + {name:'seeded-startup-failure',seed:true,owned:true,failure:true}, + {name:'seeded-configured-codex',seed:true,owned:true,configuredCodex:codexConfigured}, + {name:'seeded-reset-codex',seed:true,owned:true,configuredCodex:codexConfigured,overrides:{CODEX_HOME:''}}, + {name:'seeded-overrides',seed:true,owned:true,configuredCodex:codexConfigured,overrides:{CODEX_HOME:codexOverride,PLAYWRIGHT_BROWSERS_PATH:'0'}}, + {name:'unseeded',seed:false,owned:false}, + {name:'explicit-home',seed:true,owned:false,overrides:{HOME:operatorHome}}, + {name:'explicit-config',seed:true,owned:false,overrides:{CLAUDE_CONFIG_DIR:customConfig}}, + {name:'legacy',seed:true,owned:false,legacy:true}, + ]; + const children: Array> = []; + const records: string[] = []; + const originalSection = digest(path.join(oldRoot,section)); + try { + for (const item of cases) { + const record = path.join(dir,item.name+'.json'); records.push(record); + const worker = path.join(dir,item.name+'.ts'); + fs.writeFileSync(worker, ` +import { launchClaudePty } from ${JSON.stringify(runner)}; +const session = await launchClaudePty({seedSkills:${item.seed},timeoutMs:10000, + model:'unchanged-claude-model',env:${JSON.stringify({GSTACK_HOME:state,RUNTIME_RECORD:record,RUNTIME_FAIL:item.failure?'1':'0',...item.overrides})}}); +try { + let failed = false; + try { await session.waitFor('RUNTIME_PROBE_READY',{timeoutMs:7000,pollMs:20}); } + catch (error) { failed = true; if (!${Boolean(item.failure)}) throw error; } + if (failed !== ${Boolean(item.failure)}) throw new Error('unexpected startup outcome'); +} finally { await session.close(); } +`); + const child = Bun.spawn([process.execPath,worker], { + env:{...process.env,HOME:operatorHome,ANTHROPIC_API_KEY:'fixture-api-key',CODEX_HOME:item.configuredCodex, + PLAYWRIGHT_BROWSERS_PATH:undefined,EVALS_HERMETIC:item.legacy?'0':'1',BROWSE_TERMINAL_BINARY:fake}, + stdout:'pipe',stderr:'pipe', + }); + children.push(child); + const timer = setTimeout(() => child.kill('SIGKILL'),15000); + try { + const [code,stdout,stderr] = await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]); + expect(code,item.name+'\n'+stdout+stderr).toBe(0); + } finally { clearTimeout(timer); } + const result = JSON.parse(fs.readFileSync(record,'utf8')); + expect(result.claudeAuthPreserved).toBe(true); + expect(result.codexAuthPreserved).toBe(true); + expect(result.codexConfig).toBe('model = "unchanged-fixture-model"\n'); + expect(result.args.slice(0,4)).toEqual(['--model','unchanged-claude-model','--permission-mode','plan']); + expect(result.state).toBe(state); + expect(result.phaseExit).toBe(0); expect(result.configExit).toBe(0); + if (item.owned) { + expect(result.phaseHash).toBe(digest(path.join(ROOT,section))); + expect(result.home).not.toBe(operatorHome); + expect(result.root).toBe(fs.realpathSync(ROOT)); + expect(result.configValue).toBe('disabled'); expect(result.runtimeAllowed).toBe(true); + const expectedCodex = item.overrides?.CODEX_HOME !== undefined + ? item.overrides.CODEX_HOME || codexDefault : item.configuredCodex || codexDefault; + expect(result.codexHome).toBe(expectedCodex); + const cacheBase = process.platform === 'darwin' ? path.join(operatorHome,'Library','Caches') : path.join(operatorHome,'.cache'); + expect(result.browserCache).toBe(item.overrides?.PLAYWRIGHT_BROWSERS_PATH || path.join(cacheBase,'ms-playwright')); + expect(fs.existsSync(result.home)).toBe(false); + } else { + expect(result.home).toBe(operatorHome); expect(result.phaseHash).toBe(originalSection); + expect(result.configValue).toBe('stale-runtime'); expect(result.runtimeAllowed).toBe(false); + } + if (item.name === 'explicit-config') expect(result.configDir).toBe(customConfig); + expect(() => process.kill(result.pid,0)).toThrow(); + } + expect(digest(path.join(oldRoot,section))).toBe(originalSection); + expect(fs.readFileSync(path.join(state,'config.yaml'),'utf8')).toBe('codex_reviews: disabled\n'); + expect(fs.readdirSync(path.join(operatorHome,'.claude'))).toEqual(['skills']); + } finally { + for (const child of children) try { child.kill('SIGKILL'); } catch { /* already reaped */ } + for (const file of records) if (fs.existsSync(file)) { + try { process.kill(JSON.parse(fs.readFileSync(file,'utf8')).pid,'SIGKILL'); } catch { /* already reaped */ } + } + fs.rmSync(dir,{recursive:true,force:true}); + } + },120000); +});