diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 72a8a9dd8..0d00f1324 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -125,6 +125,9 @@ describeSol('GPT-5.6 Sol full-artifact scope termination', () => { const generated = spawnSync( 'bun', ['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'], + // LIVE-REPO CWD: gen-skill-docs --out-dir is claude-host-only, so the + // Sol render is unavoidably in-place; prior .agents tree is snapshotted + // above and restored below. { cwd: ROOT, encoding: 'utf8', timeout: 120_000 }, ); if (generated.status !== 0) { diff --git a/test/skill-e2e-auto-decide-preserved.test.ts b/test/skill-e2e-auto-decide-preserved.test.ts index 4f5d8e308..0294a742e 100644 --- a/test/skill-e2e-auto-decide-preserved.test.ts +++ b/test/skill-e2e-auto-decide-preserved.test.ts @@ -67,6 +67,8 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', () // claude would resolve). The preference file path keys on this slug. const slugBin = path.join(ROOT, 'bin', 'gstack-slug'); const slugRes = spawnSync(slugBin, [], { + // LIVE-REPO CWD: gstack-slug resolves the slug from this repo's git + // remote — must match what the spawned claude (repo cwd) resolves. cwd: ROOT, env: { ...process.env, GSTACK_HOME: tmpHome }, encoding: 'utf-8', diff --git a/test/skill-e2e-office-hours-brain-writeback.test.ts b/test/skill-e2e-office-hours-brain-writeback.test.ts index 74fbe0af6..0dc085000 100644 --- a/test/skill-e2e-office-hours-brain-writeback.test.ts +++ b/test/skill-e2e-office-hours-brain-writeback.test.ts @@ -122,6 +122,8 @@ describeIfSelected( '--respect-detection', ], { + // LIVE-REPO CWD: gen-skill-docs regenerates the in-repo + // office-hours SKILL.md + section (snapshotted/restored in finally). cwd: ROOT, env: { ...process.env, GSTACK_HOME: tmpHome }, stdio: ['ignore', 'pipe', 'pipe'], diff --git a/test/skill-e2e-opus-47.test.ts b/test/skill-e2e-opus-47.test.ts index 328ebf42a..8f4c0f3e6 100644 --- a/test/skill-e2e-opus-47.test.ts +++ b/test/skill-e2e-opus-47.test.ts @@ -67,6 +67,8 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string { const result = spawnSync( 'bun', ['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'], + // LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the + // in-repo SKILL.md files (restored to default in afterAll below). { cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 }, ); if (result.status !== 0) { @@ -169,6 +171,8 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { // whichever model ran last. Reset to the default (claude) so the tree // matches what would be checked in. spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], { + // LIVE-REPO CWD: restores the in-repo SKILL.md files to the default + // model render after mkEvalRoot's --model regens. cwd: ROOT, stdio: 'pipe', timeout: 60_000, diff --git a/test/skill-e2e-plan-ceo-finding-count.test.ts b/test/skill-e2e-plan-ceo-finding-count.test.ts index e299adc56..4c624d912 100644 --- a/test/skill-e2e-plan-ceo-finding-count.test.ts +++ b/test/skill-e2e-plan-ceo-finding-count.test.ts @@ -18,6 +18,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, ceoStep0Boundary, @@ -62,8 +64,8 @@ const N_PAIRED = 2; const FLOOR_PAIRED = 2; const CEILING_PAIRED = 4; -const PLAN_CEO_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).', +const planCeo5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Payment Processing Integration', '', @@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [ 'order in a loop.', ].join('\n'); -const PLAN_CEO_2_PAIRED_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).', +const planCeo2PairedFindings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Payment Processing — Test Coverage', '', @@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [ 'the success path is correctness, the failure path is graceful degradation.', ].join('\n'); -const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md'; -const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md'; - describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_CEO_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: PLAN_CEO_5_FINDINGS, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling - firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt: planCeo5Findings(planPath), + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling + firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } // D19: review report at bottom of plan file. - if (!fs.existsSync(PLAN_CEO_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` + `Either the agent ignored the path instruction in the follow-up prompt, or\n` + `the helper exited before the agent wrote the file. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -187,7 +188,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_CEO_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } @@ -199,24 +200,24 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () test( `paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: CEILING_PAIRED + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir — see the distinct-findings test above. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt: planCeo2PairedFindings(planPath), + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: CEILING_PAIRED + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `paired-finding control FAILED: outcome=${obs.outcome}\n` + @@ -242,7 +243,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-ceo-finding-floor.test.ts b/test/skill-e2e-plan-ceo-finding-floor.test.ts index e99da2c53..01753543e 100644 --- a/test/skill-e2e-plan-ceo-finding-floor.test.ts +++ b/test/skill-e2e-plan-ceo-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { skillName: 'plan-ceo-review', slashCommand: '/plan-ceo-review', followUpPrompt: FORCING_FLOOR_CEO, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-ceo-split-overflow.test.ts b/test/skill-e2e-plan-ceo-split-overflow.test.ts index 896c8c84b..57d39c8f2 100644 --- a/test/skill-e2e-plan-ceo-split-overflow.test.ts +++ b/test/skill-e2e-plan-ceo-split-overflow.test.ts @@ -35,6 +35,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, ceoStep0Boundary, @@ -46,30 +48,38 @@ const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option -const PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md'; +/** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt. + * Rewritten per-run to a mkdtemp path so concurrent runs (--retry, + * EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */ +const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md'; describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { test( `5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`, async () => { - try { - fs.rmSync(PLAN_PATH, { force: true }); - } catch { - /* best-effort */ + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-split-overflow-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-split-overflow.md'); + const followUpPrompt = FORCING_SPLIT_OVERFLOW_CEO.replaceAll(FIXTURE_PLAN_PATH, planPath); + if (!followUpPrompt.includes(planPath)) { + throw new Error( + `fixture drift: FORCING_SPLIT_OVERFLOW_CEO no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`, + ); } - const obs = await runPlanSkillCounting({ - skillName: 'plan-ceo-review', - slashCommand: '/plan-ceo-review', - followUpPrompt: FORCING_SPLIT_OVERFLOW_CEO, - isLastStep0AUQ: ceoStep0Boundary, - reviewCountCeiling: N + 3, // hard cap above floor + tolerance - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); - try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-ceo-review', + slashCommand: '/plan-ceo-review', + followUpPrompt, + isLastStep0AUQ: ceoStep0Boundary, + reviewCountCeiling: N + 3, // hard cap above floor + tolerance + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `split-overflow test FAILED: outcome=${obs.outcome}\n` + @@ -97,7 +107,7 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { } } finally { try { - fs.rmSync(PLAN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-design-finding-count.test.ts b/test/skill-e2e-plan-design-finding-count.test.ts index 8793dca38..4f0d8b544 100644 --- a/test/skill-e2e-plan-design-finding-count.test.ts +++ b/test/skill-e2e-plan-design-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, designStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; const CEILING = N + 2; -const PLAN_DESIGN_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-design.md (use Edit/Write to that exact path).', +const planDesign5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Settings Page UI redesign', '', @@ -50,30 +52,30 @@ const PLAN_DESIGN_5_FINDINGS = [ 'see a frozen page; we should add a spinner or skeleton state.', ].join('\n'); -const PLAN_DESIGN_PATH = '/tmp/gstack-test-plan-design.md'; - describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_DESIGN_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-design-review', - slashCommand: '/plan-design-review', - followUpPrompt: PLAN_DESIGN_5_FINDINGS, - isLastStep0AUQ: designStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-design-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-design.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-design-review', + slashCommand: '/plan-design-review', + followUpPrompt: planDesign5Findings(planPath), + isLastStep0AUQ: designStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -105,17 +107,17 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', ); } - if (!fs.existsSync(PLAN_DESIGN_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_DESIGN_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_DESIGN_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_DESIGN_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -124,7 +126,7 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', } } finally { try { - fs.rmSync(PLAN_DESIGN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index dc556f33b..3f05ab36b 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { skillName: 'plan-design-review', slashCommand: '/plan-design-review', followUpPrompt: FORCING_FLOOR_DESIGN, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index a9877922f..bdf25c75e 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -43,6 +43,8 @@ describeE2E('/plan-design-review with UI scope (gate)', () => { const session = await launchClaudePty({ permissionMode: 'plan', + // LIVE-REPO CWD: PTY session needs the repo cwd — skill registry, + // hermetic pre-trusted dir, and the repo-relative fixture path above. cwd: ROOT, timeoutMs: 720_000, seedSkills: true, diff --git a/test/skill-e2e-plan-devex-finding-count.test.ts b/test/skill-e2e-plan-devex-finding-count.test.ts index 0f1eb8fb7..3a3098b63 100644 --- a/test/skill-e2e-plan-devex-finding-count.test.ts +++ b/test/skill-e2e-plan-devex-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, devexStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; const CEILING = N + 2; -const PLAN_DEVEX_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-devex.md (use Edit/Write to that exact path).', +const planDevex5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Public SDK Beta Launch', '', @@ -50,30 +52,30 @@ const PLAN_DEVEX_5_FINDINGS = [ 'of solved problems.', ].join('\n'); -const PLAN_DEVEX_PATH = '/tmp/gstack-test-plan-devex.md'; - describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_DEVEX_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-devex-review', - slashCommand: '/plan-devex-review', - followUpPrompt: PLAN_DEVEX_5_FINDINGS, - isLastStep0AUQ: devexStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-devex-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-devex.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-devex-review', + slashCommand: '/plan-devex-review', + followUpPrompt: planDevex5Findings(planPath), + isLastStep0AUQ: devexStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -105,17 +107,17 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', ( ); } - if (!fs.existsSync(PLAN_DEVEX_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_DEVEX_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_DEVEX_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_DEVEX_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -124,7 +126,7 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', ( } } finally { try { - fs.rmSync(PLAN_DEVEX_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-devex-finding-floor.test.ts b/test/skill-e2e-plan-devex-finding-floor.test.ts index e87d7e0ed..e1da9fe31 100644 --- a/test/skill-e2e-plan-devex-finding-floor.test.ts +++ b/test/skill-e2e-plan-devex-finding-floor.test.ts @@ -19,6 +19,8 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { skillName: 'plan-devex-review', slashCommand: '/plan-devex-review', followUpPrompt: FORCING_FLOOR_DEVEX, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-eng-finding-count.test.ts b/test/skill-e2e-plan-eng-finding-count.test.ts index 257e579d9..a0f14fd6d 100644 --- a/test/skill-e2e-plan-eng-finding-count.test.ts +++ b/test/skill-e2e-plan-eng-finding-count.test.ts @@ -11,6 +11,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, engStep0Boundary, @@ -23,8 +25,8 @@ const N = 5; const FLOOR = N - 1; // 4 const CEILING = N + 2; // 7 -const PLAN_ENG_5_FINDINGS = [ - 'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-eng.md (use Edit/Write to that exact path).', +const planEng5Findings = (planPath: string) => [ + `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Multi-tenant Auth Refactor', '', @@ -49,30 +51,30 @@ const PLAN_ENG_5_FINDINGS = [ 'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.', ].join('\n'); -const PLAN_ENG_PATH = '/tmp/gstack-test-plan-eng.md'; - describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => { test( `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, async () => { - try { - fs.rmSync(PLAN_ENG_PATH, { force: true }); - } catch { - /* best-effort */ - } - - const obs = await runPlanSkillCounting({ - skillName: 'plan-eng-review', - slashCommand: '/plan-eng-review', - followUpPrompt: PLAN_ENG_5_FINDINGS, - isLastStep0AUQ: engStep0Boundary, - reviewCountCeiling: CEILING + 1, - cwd: process.cwd(), - timeoutMs: 1_500_000, - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); + // Per-run artifact dir: a hardcoded shared /tmp path collides under + // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally- + // rmSync deletes this run's artifact → spurious D19 failure). + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-eng.md'); try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-eng-review', + slashCommand: '/plan-eng-review', + followUpPrompt: planEng5Findings(planPath), + isLastStep0AUQ: engStep0Boundary, + reviewCountCeiling: CEILING + 1, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` + @@ -104,17 +106,17 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () ); } - if (!fs.existsSync(PLAN_ENG_PATH)) { + if (!fs.existsSync(planPath)) { throw new Error( - `D19 FAIL: agent did not produce expected plan file at ${PLAN_ENG_PATH}. ` + + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` + `outcome=${obs.outcome} review=${obs.reviewCount}`, ); } - const planContent = fs.readFileSync(PLAN_ENG_PATH, 'utf-8'); + const planContent = fs.readFileSync(planPath, 'utf-8'); const verdict = assertReviewReportAtBottom(planContent); if (!verdict.ok) { throw new Error( - `D19 FAIL: plan file at ${PLAN_ENG_PATH} ${verdict.reason}\n` + + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` + (verdict.trailingHeadings ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` : '') + @@ -123,7 +125,7 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () } } finally { try { - fs.rmSync(PLAN_ENG_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-plan-eng-finding-floor.test.ts b/test/skill-e2e-plan-eng-finding-floor.test.ts index f5b7cfeea..8e238f35f 100644 --- a/test/skill-e2e-plan-eng-finding-floor.test.ts +++ b/test/skill-e2e-plan-eng-finding-floor.test.ts @@ -30,6 +30,8 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { skillName: 'plan-eng-review', slashCommand: '/plan-eng-review', followUpPrompt: FORCING_FLOOR_ENG, + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), timeoutMs: 600_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, diff --git a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts index bac69496a..73e1179c6 100644 --- a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts +++ b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts @@ -27,6 +27,8 @@ import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { runPlanSkillCounting, engStep0Boundary, @@ -38,30 +40,38 @@ const describeE2E = describeE2ETier('periodic'); const N = 4; const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding -const PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md'; +/** Plan-file target baked into the FORCING_BATCHING_ENG fixture prompt. + * Rewritten per-run to a mkdtemp path so concurrent runs (--retry, + * EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */ +const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md'; describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => { test( `4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`, async () => { - try { - fs.rmSync(PLAN_PATH, { force: true }); - } catch { - /* best-effort */ + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-batching-')); + const planPath = path.join(tmpDir, 'gstack-test-plan-eng-batching.md'); + const followUpPrompt = FORCING_BATCHING_ENG.replaceAll(FIXTURE_PLAN_PATH, planPath); + if (!followUpPrompt.includes(planPath)) { + throw new Error( + `fixture drift: FORCING_BATCHING_ENG no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`, + ); } - const obs = await runPlanSkillCounting({ - skillName: 'plan-eng-review', - slashCommand: '/plan-eng-review', - followUpPrompt: FORCING_BATCHING_ENG, - isLastStep0AUQ: engStep0Boundary, - reviewCountCeiling: N + 3, // hard cap above floor + tolerance - cwd: process.cwd(), - timeoutMs: 1_500_000, // 25 min - env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, - }); - try { + const obs = await runPlanSkillCounting({ + skillName: 'plan-eng-review', + slashCommand: '/plan-eng-review', + followUpPrompt, + isLastStep0AUQ: engStep0Boundary, + reviewCountCeiling: N + 3, // hard cap above floor + tolerance + // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill + // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). + cwd: process.cwd(), + timeoutMs: 1_500_000, // 25 min + env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + }); + if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { throw new Error( `multi-finding batching test FAILED: outcome=${obs.outcome}\n` + @@ -85,7 +95,7 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () } } finally { try { - fs.rmSync(PLAN_PATH, { force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } diff --git a/test/skill-e2e-preamble-script-ab.test.ts b/test/skill-e2e-preamble-script-ab.test.ts index 7b0827c59..2a2603c8a 100644 --- a/test/skill-e2e-preamble-script-ab.test.ts +++ b/test/skill-e2e-preamble-script-ab.test.ts @@ -40,6 +40,8 @@ const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump) function inlineSkill(): string { return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, { + // LIVE-REPO CWD: git show needs this repo's history to read the + // pre-Phase-1 SKILL.md render at INLINE_REF. cwd: ROOT, encoding: 'utf-8', maxBuffer: 8 * 1024 * 1024,