test: isolate installed runtime paths in planning evals

This commit is contained in:
Garry Tan committed 2026-09-08 18:14:04 +00:00
1 parent 823bdaf40c
commit 10b6cf5b80
7 files changed
+288 -30

No files matched your search

+36
View File
@@ -87,6 +87,42 @@ const FRESH_RETRY_CAPTURE = [
].join('\n');
describe('autoplan routing setup handling', () => {
// Source-F retry's first complete frame preceded damaged terminal redraws.
const F_SETUP_CAPTURE = [
'Planning: /tmp/hermetic/.claude/plans/deep-coalescing-valiant.md',
'☐Skillrouting',
"│gstackworksbestwhenyourproject'sCLAUDE.mdincludesskillroutingrules.Addthemnow?",
'❯1.AddroutingrulestoCLAUDE.md',
'AppendsskillroutingrulestoCLAUDE.mdsogstackauto-invokestherightskillforcommonrequests(review,ship,',
'investigate,etc.).Willbecommittedtotherepo.(recommended)',
'2.Nothanks,skip',
"I'llinvokeskillsmanually.Youcanaddroutinglater.",
'3.Typesomething.',
'4.Chataboutthis',
'Entertoselect·↑/↓tonavigate·Esctocancel',
].join('\r\r');
test('answers the captured combined decline action once, regardless of option order', () => {
const seen = new Set<string>();
expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE, seen)).toBe('1\r');
expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE, seen)).toBeNull();
const reordered = F_SETUP_CAPTURE.replace('❯1.AddroutingrulestoCLAUDE.md', '❯1.Nothanks,skip')
.replace('2.Nothanks,skip', '2.AddroutingrulestoCLAUDE.md');
expect(autoplanRoutingSetupInput(reordered, new Set())).toBe('2\r');
expect(autoplanRoutingSetupInput(F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip—invoke skills manually'), new Set())).toBe('1\r');
});
test('does not infer a routing answer from damaged, ambiguous, or unrelated setup choices', () => {
for (const frame of [
F_SETUP_CAPTURE.replace('Addroutingrules', 'Addrutingrules'),
F_SETUP_CAPTURE.replace('Nothanks,skip', 'Nothank,skip'),
F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip the review'),
F_SETUP_CAPTURE.replace('Nothanks,skip', 'No thanks, skip then delete CLAUDE.md'),
F_SETUP_CAPTURE.replace('3.Typesomething.', '3.Skip'),
F_SETUP_CAPTURE.replace("gstackworksbestwhenyourproject'sCLAUDE.mdincludesskillroutingrules.Addthemnow?", 'Which routing design should the application use?'),
]) expect(autoplanRoutingSetupInput(frame, new Set()), frame).toBeNull();
});
test('answers the fresh retry manual-invocation setup once in either option order', () => {
const seen = new Set<string>();
expect(autoplanRoutingSetupInput(FRESH_RETRY_CAPTURE, seen)).toBe('1\r');
+3 -1
View File
@@ -24,7 +24,9 @@ export function autoplanRoutingSetupInput(visible: string, seen: Set<string>): s
const title = option.title.replace(/\(Recommended\)$/i, '');
const prefix = /^(?:Nothanks|Skip)(?:[,—–-])?/i.exec(title);
if (!prefix) return false;
const action = title.slice(prefix[0].length);
// 'No thanks' can be followed by the same explicit Skip action. Strip
// that decline verb before checking any optional manual-invocation text.
const action = title.slice(prefix[0].length).replace(/^skip(?:[,—–-])?/i, '');
return action === '' || /^(?:manual(?:invocation)?|(?:I['’]ll)?invoke(?:skills)?manually)$/i.test(action);
});
if (add.length !== 1 || decline.length !== 1 || add[0]!.index === decline[0]!.index) return null;
+9 -1
View File
@@ -27,6 +27,7 @@ import * as os from 'os';
import * as path from 'path';
import { stripVTControlCharacters } from 'node:util';
import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env';
import { withHermeticSkillRuntime } from './hermetic-skill-runtime';
import { createPlanCountFixture } from './plan-count-fixture';
import { createPlanCountSnapshotWriter } from './plan-count-artifacts';
import { readPlanCountTranscript, unresolvedPlanQuestionCalls, type NativePlanQuestionCall, type PlanCountTranscript } from './plan-count-transcript';
@@ -1600,9 +1601,16 @@ export async function launchClaudePty(
// Hermetic by default (test/helpers/hermetic-env.ts): operator session
// context never reaches the child; per-test opts.env merges last.
const childEnv = hermeticChildEnv(opts.env);
let childEnv = hermeticChildEnv(opts.env);
if (opts.seedSkills && hermetic && !opts.env?.CLAUDE_CONFIG_DIR) {
childEnv.CLAUDE_CONFIG_DIR = hermeticSkillsConfigDir();
if (opts.env?.HOME === undefined) {
const runtime = withHermeticSkillRuntime(childEnv);
childEnv = runtime.env;
// Lazy sections are installed runtime inputs outside the fixture cwd.
// Grant this known directory, preserving all other permission decisions.
args.push('--add-dir', runtime.root);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
+5 -6
View File
@@ -268,12 +268,11 @@ let cachedSkillsConfigDir: string | null = null;
* cover it. Ends in `/.claude` for the same plan-path anchoring reason as
* HermeticDirs.configDir.
*
* Two intentional non-hermetic edges:
* - Seeding reads the LIVE repo tree BY DESIGN — the skills ARE the subject
* under test; a snapshot would measure stale copies.
* - HOME is not hermeticized, so the ~64 absolute
* `~/.claude/skills/gstack/...` preamble references inside each SKILL.md
* still resolve to the operator install (same limitation as CI).
* Seeding reads the live repo tree: the skills are the subject under test.
* For default seeded PTY sessions, launchClaudePty also supplies an owned HOME
* via hermetic-skill-runtime so literal runtime and lazy-section paths reach
* this same checkout. Explicit HOME/config overrides remain caller-owned;
* this registration helper itself does not change their environment.
*/
export function hermeticSkillsConfigDir(): string {
if (cachedSkillsConfigDir) return cachedSkillsConfigDir;
+66
View File
@@ -0,0 +1,66 @@
/**
* Runtime paths for PTY sessions that seed the checkout's Claude skills.
*
* Claude's registry uses CLAUDE_CONFIG_DIR, but generated skills also read
* ~/.claude/skills/gstack and execute $HOME/.claude/skills/gstack/bin tools.
* Point those unchanged paths at the same checkout, never an operator install.
* This HOME is owned by the existing hermetic runRoot and its exit/GC cleanup.
*
* Two intentional shared resources survive the HOME change: nested Codex's
* configured auth/model directory, and Playwright's existing browser cache.
* Claude auth stays in the existing seeded config/API env; gstack state stays
* at the caller's GSTACK_HOME. No operator skill, hooks, or settings are copied.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { getHermeticDirs } from './hermetic-env';
let cachedRuntime: { home: string; root: string } | undefined;
export function hermeticSkillRuntime(): { home: string; root: string } {
if (cachedRuntime) return cachedRuntime;
const home = fs.mkdtempSync(path.join(getHermeticDirs().runRoot, 'skill-home-'));
const root = path.join(home, '.claude', 'skills', 'gstack');
try {
fs.mkdirSync(path.dirname(root), { recursive: true });
fs.symlinkSync(path.resolve(import.meta.dir, '..', '..'), root, 'dir');
} catch (error) {
fs.rmSync(home, { recursive: true, force: true });
throw error;
}
cachedRuntime = { home, root };
return cachedRuntime;
}
/** Keep the cache the child used before HOME isolation (Playwright defaults). */
function browserCache(env: Record<string, string>, originalHome: string): string {
if (env.PLAYWRIGHT_BROWSERS_PATH) return env.PLAYWRIGHT_BROWSERS_PATH;
const base = process.platform === 'darwin'
? path.join(originalHome, 'Library', 'Caches')
: process.platform === 'win32'
? env.LOCALAPPDATA || path.join(originalHome, 'AppData', 'Local')
: env.XDG_CACHE_HOME || path.join(originalHome, '.cache');
return path.join(base, 'ms-playwright');
}
export function withHermeticSkillRuntime(
env: Record<string, string>,
operatorEnv: NodeJS.ProcessEnv = process.env,
): { env: Record<string, string>; root: string } {
const originalHome = env.HOME || os.homedir();
const runtime = hermeticSkillRuntime();
// An explicit empty per-test value resets Codex to its original HOME default.
const codexHome = env.CODEX_HOME !== undefined
? env.CODEX_HOME || path.join(originalHome, '.codex')
: operatorEnv.CODEX_HOME || path.join(originalHome, '.codex');
return {
root: runtime.root,
env: {
...env,
HOME: runtime.home,
CODEX_HOME: codexHome,
PLAYWRIGHT_BROWSERS_PATH: browserCache(env, originalHome),
},
};
}
+22 -22
View File
@@ -100,17 +100,17 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// include question-tuning.ts and generate-ask-user-format.ts because the
// AUTO_DECIDE preamble injection lives there and changes can flip the
// regression test outcome between 'asked' and 'auto_decided'.
'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'],
'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'],
'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
// in CI these run CONCURRENT with the rest of the pty-plan-smoke suite
// (--max-concurrency + --retry 1), so worst-case cost is ~2x a single
// pass of each, sharing the API budget with sibling tests — not the
// sequential ~+10min a local read suggests.
'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'],
'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'],
// v1.21+ AskUserQuestion-blocked regression tests — Conductor launches
// claude with `--disallowedTools AskUserQuestion --permission-mode default`
@@ -120,7 +120,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// INSIDE the existing 4 plan-X-review-plan-mode test files (covered
// transitively by the entries above). Two new standalone files exist for
// skills with no prior plan-mode test:
'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'],
'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'],
'office-hours-phase4-fork': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'],
'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'],
// v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution
@@ -128,20 +128,20 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// written a never-ask preference, AUQ should still auto-decide rather than
// surfacing the question. Touches the question-tuning + preference
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'],
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'],
// Conductor → prose decision brief (Conductor signal makes prose the default;
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
// the Conductor rule, the preamble signal, the hook, and the detection helper.
'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'],
'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'],
// Real-PTY E2E batch (#6 new tests on the harness).
// Each one tests behavior the SDK harness can't observe (rendered TTY,
// numbered-option lists, multi-phase ordering, idempotency state echo).
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
@@ -154,34 +154,34 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts'],
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
// Each test drives its skill end-to-end; touchfiles include preamble +
// completion-status resolvers because they affect question cadence and
// terminal output (the regression surface this test catches).
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
// review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal
// "did agent fire ANY AUQ?" observer that exits early on first non-permission
// numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total.
'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'],
'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'],
'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'],
'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'],
'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'],
'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'],
'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'],
'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'],
// Multi-finding batching regression — periodic tier complement to the
// gate-tier finding-floor. Catches the May 2026 transcript shape where
// a model fires one AUQ then batches the rest into a "## Decisions to
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
// (it exits on first AUQ); runPlanSkillCounting can.
'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
+147
View File
@@ -0,0 +1,147 @@
/** Free installed-runtime probe through the real PTY launcher; no provider calls. */
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { createHash } from 'node:crypto';
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
const ROOT = path.resolve(import.meta.dir, '..');
const digest = (file: string) => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
describe('hermetic seeded PTY runtime', () => {
test('runtime helper and regression select every PTY consumer', () => {
const consumers = Object.entries(E2E_TOUCHFILES).filter(([, files]) => files.includes('test/helpers/claude-pty-runner.ts')).map(([name]) => name).sort();
expect(consumers.length).toBeGreaterThan(15);
for (const file of ['test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts'])
expect(selectTests([file], E2E_TOUCHFILES).selected.sort()).toEqual(consumers);
});
test.skipIf(process.platform === 'win32')('uses current lazy files and tools with scoped access, preserving auth, caches, and explicit overrides', async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-runtime-probe-'));
const operatorHome = path.join(dir, 'operator-home');
const oldRoot = path.join(operatorHome, '.claude', 'skills', 'gstack');
const section = 'autoplan/sections/ceo-phase.md';
fs.mkdirSync(path.join(oldRoot, 'autoplan', 'sections'), { recursive:true });
fs.mkdirSync(path.join(oldRoot, 'bin'));
fs.writeFileSync(path.join(oldRoot, section), 'STALE OPERATOR PHASE\n');
fs.writeFileSync(path.join(oldRoot, 'bin/gstack-config'), '#!/bin/sh\necho stale-runtime\n', { mode:0o755 });
const state = path.join(dir, 'state');
fs.mkdirSync(state);
fs.writeFileSync(path.join(state, 'config.yaml'), 'codex_reviews: disabled\n');
const codexDefault = path.join(operatorHome, '.codex');
const codexConfigured = path.join(dir, 'configured-codex');
const codexOverride = path.join(dir, 'override-codex');
for (const home of [codexDefault,codexConfigured,codexOverride]) {
fs.mkdirSync(home);
fs.writeFileSync(path.join(home,'config.toml'), 'model = "unchanged-fixture-model"\n');
fs.writeFileSync(path.join(home,'auth.json'), '{"fixture_auth":true}\n');
}
const customConfig = path.join(dir,'custom-config');
fs.mkdirSync(customConfig);
const fake = path.join(dir,'fake-claude');
fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
import * as path from 'node:path';
import { createHash } from 'node:crypto';
import { spawnSync } from 'node:child_process';
const root = path.join(process.env.HOME,'.claude','skills','gstack');
const args = process.argv.slice(2);
const addDir = args.includes('--add-dir') ? args[args.indexOf('--add-dir')+1] : null;
const phase = spawnSync('bash',['-c','cat ~/.claude/skills/gstack/autoplan/sections/ceo-phase.md'],{timeout:5000});
const config = spawnSync('bash',['-c','"$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews'],{timeout:5000});
const codexHome = process.env.CODEX_HOME || path.join(process.env.HOME,'.codex');
const record = {
pid:process.pid, home:process.env.HOME, root:fs.realpathSync(root), args,
phaseExit:phase.status, phaseHash:createHash('sha256').update(phase.stdout).digest('hex'),
configExit:config.status, configValue:config.stdout.toString().trim(),
configDir:process.env.CLAUDE_CONFIG_DIR, state:process.env.GSTACK_HOME,
runtimeAllowed:addDir !== null && fs.realpathSync(addDir) === fs.realpathSync(root),
claudeAuthPreserved:process.env.ANTHROPIC_API_KEY === 'fixture-api-key',
codexHome, codexConfig:fs.readFileSync(path.join(codexHome,'config.toml'),'utf8'),
codexAuthPreserved:JSON.parse(fs.readFileSync(path.join(codexHome,'auth.json'),'utf8')).fixture_auth === true,
browserCache:process.env.PLAYWRIGHT_BROWSERS_PATH,
};
fs.writeFileSync(process.env.RUNTIME_RECORD,JSON.stringify(record));
if (process.env.RUNTIME_FAIL === '1') throw new Error('fixture startup failure');
process.stdout.write('RUNTIME_PROBE_READY\n');
process.stdin.resume();
`);
fs.chmodSync(fake,0o755);
const runner = pathToFileURL(path.join(ROOT,'test/helpers/claude-pty-runner.ts')).href;
const cases = [
{name:'seeded-default',seed:true,owned:true},
{name:'seeded-startup-failure',seed:true,owned:true,failure:true},
{name:'seeded-configured-codex',seed:true,owned:true,configuredCodex:codexConfigured},
{name:'seeded-reset-codex',seed:true,owned:true,configuredCodex:codexConfigured,overrides:{CODEX_HOME:''}},
{name:'seeded-overrides',seed:true,owned:true,configuredCodex:codexConfigured,overrides:{CODEX_HOME:codexOverride,PLAYWRIGHT_BROWSERS_PATH:'0'}},
{name:'unseeded',seed:false,owned:false},
{name:'explicit-home',seed:true,owned:false,overrides:{HOME:operatorHome}},
{name:'explicit-config',seed:true,owned:false,overrides:{CLAUDE_CONFIG_DIR:customConfig}},
{name:'legacy',seed:true,owned:false,legacy:true},
];
const children: Array<ReturnType<typeof Bun.spawn>> = [];
const records: string[] = [];
const originalSection = digest(path.join(oldRoot,section));
try {
for (const item of cases) {
const record = path.join(dir,item.name+'.json'); records.push(record);
const worker = path.join(dir,item.name+'.ts');
fs.writeFileSync(worker, `
import { launchClaudePty } from ${JSON.stringify(runner)};
const session = await launchClaudePty({seedSkills:${item.seed},timeoutMs:10000,
model:'unchanged-claude-model',env:${JSON.stringify({GSTACK_HOME:state,RUNTIME_RECORD:record,RUNTIME_FAIL:item.failure?'1':'0',...item.overrides})}});
try {
let failed = false;
try { await session.waitFor('RUNTIME_PROBE_READY',{timeoutMs:7000,pollMs:20}); }
catch (error) { failed = true; if (!${Boolean(item.failure)}) throw error; }
if (failed !== ${Boolean(item.failure)}) throw new Error('unexpected startup outcome');
} finally { await session.close(); }
`);
const child = Bun.spawn([process.execPath,worker], {
env:{...process.env,HOME:operatorHome,ANTHROPIC_API_KEY:'fixture-api-key',CODEX_HOME:item.configuredCodex,
PLAYWRIGHT_BROWSERS_PATH:undefined,EVALS_HERMETIC:item.legacy?'0':'1',BROWSE_TERMINAL_BINARY:fake},
stdout:'pipe',stderr:'pipe',
});
children.push(child);
const timer = setTimeout(() => child.kill('SIGKILL'),15000);
try {
const [code,stdout,stderr] = await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);
expect(code,item.name+'\n'+stdout+stderr).toBe(0);
} finally { clearTimeout(timer); }
const result = JSON.parse(fs.readFileSync(record,'utf8'));
expect(result.claudeAuthPreserved).toBe(true);
expect(result.codexAuthPreserved).toBe(true);
expect(result.codexConfig).toBe('model = "unchanged-fixture-model"\n');
expect(result.args.slice(0,4)).toEqual(['--model','unchanged-claude-model','--permission-mode','plan']);
expect(result.state).toBe(state);
expect(result.phaseExit).toBe(0); expect(result.configExit).toBe(0);
if (item.owned) {
expect(result.phaseHash).toBe(digest(path.join(ROOT,section)));
expect(result.home).not.toBe(operatorHome);
expect(result.root).toBe(fs.realpathSync(ROOT));
expect(result.configValue).toBe('disabled'); expect(result.runtimeAllowed).toBe(true);
const expectedCodex = item.overrides?.CODEX_HOME !== undefined
? item.overrides.CODEX_HOME || codexDefault : item.configuredCodex || codexDefault;
expect(result.codexHome).toBe(expectedCodex);
const cacheBase = process.platform === 'darwin' ? path.join(operatorHome,'Library','Caches') : path.join(operatorHome,'.cache');
expect(result.browserCache).toBe(item.overrides?.PLAYWRIGHT_BROWSERS_PATH || path.join(cacheBase,'ms-playwright'));
expect(fs.existsSync(result.home)).toBe(false);
} else {
expect(result.home).toBe(operatorHome); expect(result.phaseHash).toBe(originalSection);
expect(result.configValue).toBe('stale-runtime'); expect(result.runtimeAllowed).toBe(false);
}
if (item.name === 'explicit-config') expect(result.configDir).toBe(customConfig);
expect(() => process.kill(result.pid,0)).toThrow();
}
expect(digest(path.join(oldRoot,section))).toBe(originalSection);
expect(fs.readFileSync(path.join(state,'config.yaml'),'utf8')).toBe('codex_reviews: disabled\n');
expect(fs.readdirSync(path.join(operatorHome,'.claude'))).toEqual(['skills']);
} finally {
for (const child of children) try { child.kill('SIGKILL'); } catch { /* already reaped */ }
for (const file of records) if (fs.existsSync(file)) {
try { process.kill(JSON.parse(fs.readFileSync(file,'utf8')).pid,'SIGKILL'); } catch { /* already reaped */ }
}
fs.rmSync(dir,{recursive:true,force:true});
}
},120000);
});