From ae5ffefb9aad0399b06cd05e435f8ec1c223711b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 04:59:48 +0000 Subject: [PATCH] fix(test): unique tmp dirs for plan artifacts + audited live-repo cwd sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six paid PTY tests wrote their expected plan artifact to a FIXED shared /tmp path ('/tmp/gstack-test-plan-.md') and rmSync'd it in finally — under --retry 1, EVALS_JOBS>1, or two concurrent worktrees, a sibling's cleanup deletes this run's artifact and the D19 'agent did not produce expected plan file' assertion fires spuriously. Each test now mkdtemps its own dir, interpolates the unique path into the agent prompt (fixture-sourced prompts get a replaceAll + drift guard that throws if the fixture's literal ever moves), and cleans up its own dir. The 18 cwd:-into-the-live-repo sites were audited: all deliberate (skill registry + hermetic pre-trusted dir, in-repo gen renders, git history reads, slug resolution) — each now carries a '// LIVE-REPO CWD: ' comment so the next audit can tell deliberate from accidental. Co-Authored-By: Claude Fable 5 --- test/codex-e2e-sol-scope.test.ts | 3 + test/skill-e2e-auto-decide-preserved.test.ts | 2 + ...l-e2e-office-hours-brain-writeback.test.ts | 2 + test/skill-e2e-opus-47.test.ts | 4 + test/skill-e2e-plan-ceo-finding-count.test.ts | 93 ++++++++++--------- test/skill-e2e-plan-ceo-finding-floor.test.ts | 2 + .../skill-e2e-plan-ceo-split-overflow.test.ts | 44 +++++---- ...kill-e2e-plan-design-finding-count.test.ts | 52 ++++++----- ...kill-e2e-plan-design-finding-floor.test.ts | 2 + test/skill-e2e-plan-design-with-ui.test.ts | 2 + ...skill-e2e-plan-devex-finding-count.test.ts | 52 ++++++----- ...skill-e2e-plan-devex-finding-floor.test.ts | 2 + test/skill-e2e-plan-eng-finding-count.test.ts | 52 ++++++----- test/skill-e2e-plan-eng-finding-floor.test.ts | 2 + ...2e-plan-eng-multi-finding-batching.test.ts | 44 +++++---- test/skill-e2e-preamble-script-ab.test.ts | 2 + 16 files changed, 205 insertions(+), 155 deletions(-) diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 72a8a9dd8..0d00f1324 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -125,6 +125,9 @@ describeSol('GPT-5.6 Sol full-artifact scope termination', () => { const generated = spawnSync( 'bun', ['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'], + // LIVE-REPO CWD: gen-skill-docs --out-dir is claude-host-only, so the + // Sol render is unavoidably in-place; prior .agents tree is snapshotted + // above and restored below. { cwd: ROOT, encoding: 'utf8', timeout: 120_000 }, ); if (generated.status !== 0) { diff --git a/test/skill-e2e-auto-decide-preserved.test.ts b/test/skill-e2e-auto-decide-preserved.test.ts index 4f5d8e308..0294a742e 100644 --- a/test/skill-e2e-auto-decide-preserved.test.ts +++ b/test/skill-e2e-auto-decide-preserved.test.ts @@ -67,6 +67,8 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', () // claude would resolve). The preference file path keys on this slug. const slugBin = path.join(ROOT, 'bin', 'gstack-slug'); const slugRes = spawnSync(slugBin, [], { + // LIVE-REPO CWD: gstack-slug resolves the slug from this repo's git + // remote — must match what the spawned claude (repo cwd) resolves. cwd: ROOT, env: { ...process.env, GSTACK_HOME: tmpHome }, encoding: 'utf-8', diff --git a/test/skill-e2e-office-hours-brain-writeback.test.ts b/test/skill-e2e-office-hours-brain-writeback.test.ts index 74fbe0af6..0dc085000 100644 --- a/test/skill-e2e-office-hours-brain-writeback.test.ts +++ b/test/skill-e2e-office-hours-brain-writeback.test.ts @@ -122,6 +122,8 @@ describeIfSelected( '--respect-detection', ], { + // LIVE-REPO CWD: gen-skill-docs regenerates the in-repo + // office-hours SKILL.md + section (snapshotted/restored in finally). cwd: ROOT, env: { ...process.env, GSTACK_HOME: tmpHome }, stdio: ['ignore', 'pipe', 'pipe'], diff --git a/test/skill-e2e-opus-47.test.ts b/test/skill-e2e-opus-47.test.ts index 328ebf42a..8f4c0f3e6 100644 --- a/test/skill-e2e-opus-47.test.ts +++ b/test/skill-e2e-opus-47.test.ts @@ -67,6 +67,8 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string { const result = spawnSync( 'bun', ['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'], + // LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the + // in-repo SKILL.md files (restored to default in afterAll below). { cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 }, ); if (result.status !== 0) { @@ -169,6 +171,8 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { // whichever model ran last. Reset to the default (claude) so the tree // matches what would be checked in. spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], { + // LIVE-REPO CWD: restores the in-repo SKILL.md files to the default + // model render after mkEvalRoot's --model regens. cwd: ROOT, stdio: 'pipe', timeout: 60_000, diff --git a/test/skill-e2e-plan-ceo-finding-count.test.ts b/test/skill-e2e-plan-ceo-finding-count.test.ts index e299adc56..4c624d912 100644 --- a/test/skill-e2e-plan-ceo-finding-count.test.ts +++ b/test/skill-e2e-plan-ceo-finding-count.test.ts @@ -18,6 +18,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, ceoStep0Boundary, @@ -62,8 +64,8 @@ const N_PAIRED = 2; const FLOOR_PAIRED = 2; const CEILING_PAIRED = 4; -const PLAN_CEO_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).', +const planCeo5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Payment Processing Integration', '', @@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [ 'order in a loop.', ].join('\n'); -const PLAN_CEO_2_PAIRED_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).', +const planCeo2PairedFindings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Payment Processing — Test Coverage', '', @@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [ 'the success path is correctness, the failure path is graceful degradation.', ].join('\n'); -const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md'; -const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md'; - describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_CEO_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: PLAN_CEO_5_FINDINGS, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling - firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt: planCeo5Findings(planPath), + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling + firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } // D19: review report at bottom of plan file. - if (!fs.existsSync(PLAN_CEO_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` + `Either the agent ignored the path instruction in the follow-up prompt, or\n` + `the helper exited before the agent wrote the file. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -187,7 +188,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_CEO_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } @@ -199,24 +200,24 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () test( `paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: CEILING_PAIRED + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir — see the distinct-findings test above. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt: planCeo2PairedFindings(planPath), + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: CEILING_PAIRED + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `paired-finding control FAILED: outcome=${obs.outcome}\n` + @@ -242,7 +243,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-ceo-finding-floor.test.ts b/test/skill-e2e-plan-ceo-finding-floor.test.ts index e99da2c53..01753543e 100644 --- a/test/skill-e2e-plan-ceo-finding-floor.test.ts +++ b/test/skill-e2e-plan-ceo-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { skillName: 'plan-ceo-review', slashCommand: '/plan-ceo-review', followUpPrompt: FORCING_FLOOR_CEO, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-ceo-split-overflow.test.ts b/test/skill-e2e-plan-ceo-split-overflow.test.ts index 896c8c84b..57d39c8f2 100644 --- a/test/skill-e2e-plan-ceo-split-overflow.test.ts +++ b/test/skill-e2e-plan-ceo-split-overflow.test.ts @@ -35,6 +35,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, ceoStep0Boundary, @@ -46,30 +48,38 @@ const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option -const PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md'; +/** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt. + * Rewritten per-run to a mkdtemp path so concurrent runs (--retry, + * EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */ +const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md'; describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { test( `5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`, async () => { - try { - fs.rmSync(PLAN_PATH, { force: true }); - } catch { - /* best-effort */ + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-split-overflow-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-split-overflow.md'); + const followUpPrompt = FORCING_SPLIT_OVERFLOW_CEO.replaceAll(FIXTURE_PLAN_PATH, planPath); + if (!followUpPrompt.includes(planPath)) { + throw new Error( + `fixture drift: FORCING_SPLIT_OVERFLOW_CEO no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`, + ); } - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: FORCING_SPLIT_OVERFLOW_CEO, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: N + 3, // hard cap above floor + tolerance - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); - try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt, + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: N + 3, // hard cap above floor + tolerance + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `split-overflow test FAILED: outcome=${obs.outcome}\n` + @@ -97,7 +107,7 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { } } finally { try { - fs.rmSync(PLAN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-design-finding-count.test.ts b/test/skill-e2e-plan-design-finding-count.test.ts index 8793dca38..4f0d8b544 100644 --- a/test/skill-e2e-plan-design-finding-count.test.ts +++ b/test/skill-e2e-plan-design-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, designStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; const CEILING = N + 2; -const PLAN_DESIGN_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-design.md (use Edit/Write to that exact path).', +const planDesign5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Settings Page UI redesign', '', @@ -50,30 +52,30 @@ const PLAN_DESIGN_5_FINDINGS = [ 'see a frozen page; we should add a spinner or skeleton state.', ].join('\n'); -const PLAN_DESIGN_PATH = '/tmp/gstack-test-plan-design.md'; - describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_DESIGN_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-design-review', - slashCommand: '/plan-design-review', - followUpPrompt: PLAN_DESIGN_5_FINDINGS, - isLastStep0AUQ: designStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-design-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-design.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-design-review', + slashCommand: '/plan-design-review', + followUpPrompt: planDesign5Findings(planPath), + isLastStep0AUQ: designStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -105,17 +107,17 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', ); } - if (!fs.existsSync(PLAN_DESIGN_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_DESIGN_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_DESIGN_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_DESIGN_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -124,7 +126,7 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', } } finally { try { - fs.rmSync(PLAN_DESIGN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index dc556f33b..3f05ab36b 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { skillName: 'plan-design-review', slashCommand: '/plan-design-review', followUpPrompt: FORCING_FLOOR_DESIGN, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index a9877922f..bdf25c75e 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -43,6 +43,8 @@ describeE2E('/plan-design-review with UI scope (gate)', () => { const session = await launchClaudePty({ permissionMode: 'plan', + // LIVE-REPO CWD: PTY session needs the repo cwd — skill registry, + // hermetic pre-trusted dir, and the repo-relative fixture path above. cwd: ROOT, timeoutMs: 720_000, seedSkills: true, diff --git a/test/skill-e2e-plan-devex-finding-count.test.ts b/test/skill-e2e-plan-devex-finding-count.test.ts index 0f1eb8fb7..3a3098b63 100644 --- a/test/skill-e2e-plan-devex-finding-count.test.ts +++ b/test/skill-e2e-plan-devex-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, devexStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; const CEILING = N + 2; -const PLAN_DEVEX_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-devex.md (use Edit/Write to that exact path).', +const planDevex5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Public SDK Beta Launch', '', @@ -50,30 +52,30 @@ const PLAN_DEVEX_5_FINDINGS = [ 'of solved problems.', ].join('\n'); -const PLAN_DEVEX_PATH = '/tmp/gstack-test-plan-devex.md'; - describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_DEVEX_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-devex-review', - slashCommand: '/plan-devex-review', - followUpPrompt: PLAN_DEVEX_5_FINDINGS, - isLastStep0AUQ: devexStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-devex-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-devex.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-devex-review', + slashCommand: '/plan-devex-review', + followUpPrompt: planDevex5Findings(planPath), + isLastStep0AUQ: devexStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -105,17 +107,17 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', ( ); } - if (!fs.existsSync(PLAN_DEVEX_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_DEVEX_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_DEVEX_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_DEVEX_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -124,7 +126,7 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', ( } } finally { try { - fs.rmSync(PLAN_DEVEX_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-devex-finding-floor.test.ts b/test/skill-e2e-plan-devex-finding-floor.test.ts index e87d7e0ed..e1da9fe31 100644 --- a/test/skill-e2e-plan-devex-finding-floor.test.ts +++ b/test/skill-e2e-plan-devex-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { skillName: 'plan-devex-review', slashCommand: '/plan-devex-review', followUpPrompt: FORCING_FLOOR_DEVEX, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-eng-finding-count.test.ts b/test/skill-e2e-plan-eng-finding-count.test.ts index 257e579d9..a0f14fd6d 100644 --- a/test/skill-e2e-plan-eng-finding-count.test.ts +++ b/test/skill-e2e-plan-eng-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, engStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; // 4 const CEILING = N + 2; // 7 -const PLAN_ENG_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-eng.md (use Edit/Write to that exact path).', +const planEng5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Multi-tenant Auth Refactor', '', @@ -49,30 +51,30 @@ const PLAN_ENG_5_FINDINGS = [ 'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.', ].join('\n'); -const PLAN_ENG_PATH = '/tmp/gstack-test-plan-eng.md'; - describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_ENG_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-eng-review', - slashCommand: '/plan-eng-review', - followUpPrompt: PLAN_ENG_5_FINDINGS, - isLastStep0AUQ: engStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-eng.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-eng-review', + slashCommand: '/plan-eng-review', + followUpPrompt: planEng5Findings(planPath), + isLastStep0AUQ: engStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -104,17 +106,17 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () ); } - if (!fs.existsSync(PLAN_ENG_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_ENG_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_ENG_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_ENG_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -123,7 +125,7 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_ENG_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-eng-finding-floor.test.ts b/test/skill-e2e-plan-eng-finding-floor.test.ts index f5b7cfeea..8e238f35f 100644 --- a/test/skill-e2e-plan-eng-finding-floor.test.ts +++ b/test/skill-e2e-plan-eng-finding-floor.test.ts @@ -30,6 +30,8 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { skillName: 'plan-eng-review', slashCommand: '/plan-eng-review', followUpPrompt: FORCING_FLOOR_ENG, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts index bac69496a..73e1179c6 100644 --- a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts +++ b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts @@ -27,6 +27,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, engStep0Boundary, @@ -38,30 +40,38 @@ const describeE2E = describeE2ETier('periodic'); const N = 4; const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding -const PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md'; +/** Plan-file target baked into the FORCING_BATCHING_ENG fixture prompt. + * Rewritten per-run to a mkdtemp path so concurrent runs (--retry, + * EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */ +const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md'; describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => { test( `4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`, async () => { - try { - fs.rmSync(PLAN_PATH, { force: true }); - } catch { - /* best-effort */ + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-batching-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-eng-batching.md'); + const followUpPrompt = FORCING_BATCHING_ENG.replaceAll(FIXTURE_PLAN_PATH, planPath); + if (!followUpPrompt.includes(planPath)) { + throw new Error( + `fixture drift: FORCING_BATCHING_ENG no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`, + ); } - const obs = await runPlanSkillCounting({ - skillName: 'plan-eng-review', - slashCommand: '/plan-eng-review', - followUpPrompt: FORCING_BATCHING_ENG, - isLastStep0AUQ: engStep0Boundary, - reviewCountCeiling: N + 3, // hard cap above floor + tolerance - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); - try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-eng-review', + slashCommand: '/plan-eng-review', + followUpPrompt, + isLastStep0AUQ: engStep0Boundary, + reviewCountCeiling: N + 3, // hard cap above floor + tolerance + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `multi-finding batching test FAILED: outcome=${obs.outcome}\n` + @@ -85,7 +95,7 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () } } finally { try { - fs.rmSync(PLAN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-preamble-script-ab.test.ts b/test/skill-e2e-preamble-script-ab.test.ts index 7b0827c59..2a2603c8a 100644 --- a/test/skill-e2e-preamble-script-ab.test.ts +++ b/test/skill-e2e-preamble-script-ab.test.ts @@ -40,6 +40,8 @@ const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump) function inlineSkill(): string { return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, { + // LIVE-REPO CWD: git show needs this repo's history to read the + // pre-Phase-1 SKILL.md render at INLINE_REF. cwd: ROOT, encoding: 'utf-8', maxBuffer: 8 * 1024 * 1024,