fix(test): unique tmp dirs for plan artifacts + audited live-repo cwd sites

Six paid PTY tests wrote their expected plan artifact to a FIXED shared
/tmp path ('/tmp/gstack-test-plan-<mode>.md') and rmSync'd it in
finally — under --retry 1, EVALS_JOBS>1, or two concurrent worktrees, a
sibling's cleanup deletes this run's artifact and the D19 'agent did
not produce expected plan file' assertion fires spuriously. Each test
now mkdtemps its own dir, interpolates the unique path into the agent
prompt (fixture-sourced prompts get a replaceAll + drift guard that
throws if the fixture's literal ever moves), and cleans up its own dir.

The 18 cwd:-into-the-live-repo sites were audited: all deliberate
(skill registry + hermetic pre-trusted dir, in-repo gen renders, git
history reads, slug resolution) — each now carries a
'// LIVE-REPO CWD: <reason>' comment so the next audit can tell
deliberate from accidental.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 04:59:48 +00:00
co-authored by Claude Fable 5
parent 0d2f703f28
commit ae5ffefb9a
16 changed files with 205 additions and 155 deletions
+3
View File
@@ -125,6 +125,9 @@ describeSol('GPT-5.6 Sol full-artifact scope termination', () => {
const generated = spawnSync( const generated = spawnSync(
'bun', 'bun',
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'], ['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'],
// LIVE-REPO CWD: gen-skill-docs --out-dir is claude-host-only, so the
// Sol render is unavoidably in-place; prior .agents tree is snapshotted
// above and restored below.
{ cwd: ROOT, encoding: 'utf8', timeout: 120_000 }, { cwd: ROOT, encoding: 'utf8', timeout: 120_000 },
); );
if (generated.status !== 0) { if (generated.status !== 0) {
@@ -67,6 +67,8 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
// claude would resolve). The preference file path keys on this slug. // claude would resolve). The preference file path keys on this slug.
const slugBin = path.join(ROOT, 'bin', 'gstack-slug'); const slugBin = path.join(ROOT, 'bin', 'gstack-slug');
const slugRes = spawnSync(slugBin, [], { const slugRes = spawnSync(slugBin, [], {
// LIVE-REPO CWD: gstack-slug resolves the slug from this repo's git
// remote — must match what the spawned claude (repo cwd) resolves.
cwd: ROOT, cwd: ROOT,
env: { ...process.env, GSTACK_HOME: tmpHome }, env: { ...process.env, GSTACK_HOME: tmpHome },
encoding: 'utf-8', encoding: 'utf-8',
@@ -122,6 +122,8 @@ describeIfSelected(
'--respect-detection', '--respect-detection',
], ],
{ {
// LIVE-REPO CWD: gen-skill-docs regenerates the in-repo
// office-hours SKILL.md + section (snapshotted/restored in finally).
cwd: ROOT, cwd: ROOT,
env: { ...process.env, GSTACK_HOME: tmpHome }, env: { ...process.env, GSTACK_HOME: tmpHome },
stdio: ['ignore', 'pipe', 'pipe'], stdio: ['ignore', 'pipe', 'pipe'],
+4
View File
@@ -67,6 +67,8 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
const result = spawnSync( const result = spawnSync(
'bun', 'bun',
['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'], ['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'],
// LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the
// in-repo SKILL.md files (restored to default in afterAll below).
{ cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 }, { cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 },
); );
if (result.status !== 0) { if (result.status !== 0) {
@@ -169,6 +171,8 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
// whichever model ran last. Reset to the default (claude) so the tree // whichever model ran last. Reset to the default (claude) so the tree
// matches what would be checked in. // matches what would be checked in.
spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], { spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], {
// LIVE-REPO CWD: restores the in-repo SKILL.md files to the default
// model render after mkEvalRoot's --model regens.
cwd: ROOT, cwd: ROOT,
stdio: 'pipe', stdio: 'pipe',
timeout: 60_000, timeout: 60_000,
+47 -46
View File
@@ -18,6 +18,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
ceoStep0Boundary, ceoStep0Boundary,
@@ -62,8 +64,8 @@ const N_PAIRED = 2;
const FLOOR_PAIRED = 2; const FLOOR_PAIRED = 2;
const CEILING_PAIRED = 4; const CEILING_PAIRED = 4;
const PLAN_CEO_5_FINDINGS = [ const planCeo5Findings = (planPath: string) => [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).', `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'', '',
'# Plan: Payment Processing Integration', '# Plan: Payment Processing Integration',
'', '',
@@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [
'order in a loop.', 'order in a loop.',
].join('\n'); ].join('\n');
const PLAN_CEO_2_PAIRED_FINDINGS = [ const planCeo2PairedFindings = (planPath: string) => [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).', `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'', '',
'# Plan: Payment Processing — Test Coverage', '# Plan: Payment Processing — Test Coverage',
'', '',
@@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [
'the success path is correctness, the failure path is graceful degradation.', 'the success path is correctness, the failure path is graceful degradation.',
].join('\n'); ].join('\n');
const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md';
const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md';
describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => { describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => {
test( test(
`5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`, `5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`,
async () => { async () => {
try { // Per-run artifact dir: a hardcoded shared /tmp path collides under
fs.rmSync(PLAN_CEO_PATH, { force: true }); // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
} catch { // rmSync deletes this run's artifact → spurious D19 failure).
/* best-effort */ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-'));
} const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md');
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: PLAN_CEO_5_FINDINGS,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: planCeo5Findings(planPath),
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` + `plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
} }
// D19: review report at bottom of plan file. // D19: review report at bottom of plan file.
if (!fs.existsSync(PLAN_CEO_PATH)) { if (!fs.existsSync(planPath)) {
throw new Error( throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` + `D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` +
`Either the agent ignored the path instruction in the follow-up prompt, or\n` + `Either the agent ignored the path instruction in the follow-up prompt, or\n` +
`the helper exited before the agent wrote the file. ` + `the helper exited before the agent wrote the file. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`, `outcome=${obs.outcome} review=${obs.reviewCount}`,
); );
} }
const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8'); const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent); const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) { if (!verdict.ok) {
throw new Error( throw new Error(
`D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings (verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') + : '') +
@@ -187,7 +188,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_CEO_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -199,24 +200,24 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
test( test(
`paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`, `paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`,
async () => { async () => {
try { // Per-run artifact dir — see the distinct-findings test above.
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-'));
} catch { const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md');
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_PAIRED + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: planCeo2PairedFindings(planPath),
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_PAIRED + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`paired-finding control FAILED: outcome=${obs.outcome}\n` + `paired-finding control FAILED: outcome=${obs.outcome}\n` +
@@ -242,7 +243,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -19,6 +19,8 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
skillName: 'plan-ceo-review', skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review', slashCommand: '/plan-ceo-review',
followUpPrompt: FORCING_FLOOR_CEO, followUpPrompt: FORCING_FLOOR_CEO,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: 600_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
+27 -17
View File
@@ -35,6 +35,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
ceoStep0Boundary, ceoStep0Boundary,
@@ -46,30 +48,38 @@ const describeE2E = describeE2ETier('periodic');
const N = 5; const N = 5;
const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option
const PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md'; /** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt.
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md';
describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
test( test(
`5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`, `5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`,
async () => { async () => {
try { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-split-overflow-'));
fs.rmSync(PLAN_PATH, { force: true }); const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-split-overflow.md');
} catch { const followUpPrompt = FORCING_SPLIT_OVERFLOW_CEO.replaceAll(FIXTURE_PLAN_PATH, planPath);
/* best-effort */ if (!followUpPrompt.includes(planPath)) {
throw new Error(
`fixture drift: FORCING_SPLIT_OVERFLOW_CEO no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
);
} }
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: FORCING_SPLIT_OVERFLOW_CEO,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`split-overflow test FAILED: outcome=${obs.outcome}\n` + `split-overflow test FAILED: outcome=${obs.outcome}\n` +
@@ -97,7 +107,7 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -11,6 +11,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
designStep0Boundary, designStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1; const FLOOR = N - 1;
const CEILING = N + 2; const CEILING = N + 2;
const PLAN_DESIGN_5_FINDINGS = [ const planDesign5Findings = (planPath: string) => [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-design.md (use Edit/Write to that exact path).', `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'', '',
'# Plan: Settings Page UI redesign', '# Plan: Settings Page UI redesign',
'', '',
@@ -50,30 +52,30 @@ const PLAN_DESIGN_5_FINDINGS = [
'see a frozen page; we should add a spinner or skeleton state.', 'see a frozen page; we should add a spinner or skeleton state.',
].join('\n'); ].join('\n');
const PLAN_DESIGN_PATH = '/tmp/gstack-test-plan-design.md';
describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => { describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => {
test( test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => { async () => {
try { // Per-run artifact dir: a hardcoded shared /tmp path collides under
fs.rmSync(PLAN_DESIGN_PATH, { force: true }); // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
} catch { // rmSync deletes this run's artifact → spurious D19 failure).
/* best-effort */ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-design-'));
} const planPath = path.join(tmpDir, 'gstack-test-plan-design.md');
const obs = await runPlanSkillCounting({
skillName: 'plan-design-review',
slashCommand: '/plan-design-review',
followUpPrompt: PLAN_DESIGN_5_FINDINGS,
isLastStep0AUQ: designStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-design-review',
slashCommand: '/plan-design-review',
followUpPrompt: planDesign5Findings(planPath),
isLastStep0AUQ: designStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` + `plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -105,17 +107,17 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
); );
} }
if (!fs.existsSync(PLAN_DESIGN_PATH)) { if (!fs.existsSync(planPath)) {
throw new Error( throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DESIGN_PATH}. ` + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`, `outcome=${obs.outcome} review=${obs.reviewCount}`,
); );
} }
const planContent = fs.readFileSync(PLAN_DESIGN_PATH, 'utf-8'); const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent); const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) { if (!verdict.ok) {
throw new Error( throw new Error(
`D19 FAIL: plan file at ${PLAN_DESIGN_PATH} ${verdict.reason}\n` + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings (verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') + : '') +
@@ -124,7 +126,7 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_DESIGN_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -19,6 +19,8 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
skillName: 'plan-design-review', skillName: 'plan-design-review',
slashCommand: '/plan-design-review', slashCommand: '/plan-design-review',
followUpPrompt: FORCING_FLOOR_DESIGN, followUpPrompt: FORCING_FLOOR_DESIGN,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: 600_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
@@ -43,6 +43,8 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
const session = await launchClaudePty({ const session = await launchClaudePty({
permissionMode: 'plan', permissionMode: 'plan',
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
// hermetic pre-trusted dir, and the repo-relative fixture path above.
cwd: ROOT, cwd: ROOT,
timeoutMs: 720_000, timeoutMs: 720_000,
seedSkills: true, seedSkills: true,
+27 -25
View File
@@ -11,6 +11,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
devexStep0Boundary, devexStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1; const FLOOR = N - 1;
const CEILING = N + 2; const CEILING = N + 2;
const PLAN_DEVEX_5_FINDINGS = [ const planDevex5Findings = (planPath: string) => [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-devex.md (use Edit/Write to that exact path).', `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'', '',
'# Plan: Public SDK Beta Launch', '# Plan: Public SDK Beta Launch',
'', '',
@@ -50,30 +52,30 @@ const PLAN_DEVEX_5_FINDINGS = [
'of solved problems.', 'of solved problems.',
].join('\n'); ].join('\n');
const PLAN_DEVEX_PATH = '/tmp/gstack-test-plan-devex.md';
describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => { describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => {
test( test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => { async () => {
try { // Per-run artifact dir: a hardcoded shared /tmp path collides under
fs.rmSync(PLAN_DEVEX_PATH, { force: true }); // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
} catch { // rmSync deletes this run's artifact → spurious D19 failure).
/* best-effort */ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-devex-'));
} const planPath = path.join(tmpDir, 'gstack-test-plan-devex.md');
const obs = await runPlanSkillCounting({
skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review',
followUpPrompt: PLAN_DEVEX_5_FINDINGS,
isLastStep0AUQ: devexStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review',
followUpPrompt: planDevex5Findings(planPath),
isLastStep0AUQ: devexStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` + `plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -105,17 +107,17 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
); );
} }
if (!fs.existsSync(PLAN_DEVEX_PATH)) { if (!fs.existsSync(planPath)) {
throw new Error( throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DEVEX_PATH}. ` + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`, `outcome=${obs.outcome} review=${obs.reviewCount}`,
); );
} }
const planContent = fs.readFileSync(PLAN_DEVEX_PATH, 'utf-8'); const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent); const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) { if (!verdict.ok) {
throw new Error( throw new Error(
`D19 FAIL: plan file at ${PLAN_DEVEX_PATH} ${verdict.reason}\n` + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings (verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') + : '') +
@@ -124,7 +126,7 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_DEVEX_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -19,6 +19,8 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
skillName: 'plan-devex-review', skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review', slashCommand: '/plan-devex-review',
followUpPrompt: FORCING_FLOOR_DEVEX, followUpPrompt: FORCING_FLOOR_DEVEX,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: 600_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
+27 -25
View File
@@ -11,6 +11,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
engStep0Boundary, engStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1; // 4 const FLOOR = N - 1; // 4
const CEILING = N + 2; // 7 const CEILING = N + 2; // 7
const PLAN_ENG_5_FINDINGS = [ const planEng5Findings = (planPath: string) => [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-eng.md (use Edit/Write to that exact path).', `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'', '',
'# Plan: Multi-tenant Auth Refactor', '# Plan: Multi-tenant Auth Refactor',
'', '',
@@ -49,30 +51,30 @@ const PLAN_ENG_5_FINDINGS = [
'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.', 'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.',
].join('\n'); ].join('\n');
const PLAN_ENG_PATH = '/tmp/gstack-test-plan-eng.md';
describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => { describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => {
test( test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`, `5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => { async () => {
try { // Per-run artifact dir: a hardcoded shared /tmp path collides under
fs.rmSync(PLAN_ENG_PATH, { force: true }); // --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
} catch { // rmSync deletes this run's artifact → spurious D19 failure).
/* best-effort */ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-'));
} const planPath = path.join(tmpDir, 'gstack-test-plan-eng.md');
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: PLAN_ENG_5_FINDINGS,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: planEng5Findings(planPath),
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` + `plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -104,17 +106,17 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
); );
} }
if (!fs.existsSync(PLAN_ENG_PATH)) { if (!fs.existsSync(planPath)) {
throw new Error( throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_ENG_PATH}. ` + `D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`, `outcome=${obs.outcome} review=${obs.reviewCount}`,
); );
} }
const planContent = fs.readFileSync(PLAN_ENG_PATH, 'utf-8'); const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent); const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) { if (!verdict.ok) {
throw new Error( throw new Error(
`D19 FAIL: plan file at ${PLAN_ENG_PATH} ${verdict.reason}\n` + `D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings (verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n` ? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') + : '') +
@@ -123,7 +125,7 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_ENG_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -30,6 +30,8 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
skillName: 'plan-eng-review', skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review', slashCommand: '/plan-eng-review',
followUpPrompt: FORCING_FLOOR_ENG, followUpPrompt: FORCING_FLOOR_ENG,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: 600_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
@@ -27,6 +27,8 @@
import { test } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { import {
runPlanSkillCounting, runPlanSkillCounting,
engStep0Boundary, engStep0Boundary,
@@ -38,30 +40,38 @@ const describeE2E = describeE2ETier('periodic');
const N = 4; const N = 4;
const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding
const PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md'; /** Plan-file target baked into the FORCING_BATCHING_ENG fixture prompt.
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md';
describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => { describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => {
test( test(
`4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`, `4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`,
async () => { async () => {
try { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-batching-'));
fs.rmSync(PLAN_PATH, { force: true }); const planPath = path.join(tmpDir, 'gstack-test-plan-eng-batching.md');
} catch { const followUpPrompt = FORCING_BATCHING_ENG.replaceAll(FIXTURE_PLAN_PATH, planPath);
/* best-effort */ if (!followUpPrompt.includes(planPath)) {
throw new Error(
`fixture drift: FORCING_BATCHING_ENG no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
);
} }
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: FORCING_BATCHING_ENG,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try { try {
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) { if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error( throw new Error(
`multi-finding batching test FAILED: outcome=${obs.outcome}\n` + `multi-finding batching test FAILED: outcome=${obs.outcome}\n` +
@@ -85,7 +95,7 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', ()
} }
} finally { } finally {
try { try {
fs.rmSync(PLAN_PATH, { force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} catch { } catch {
/* best-effort */ /* best-effort */
} }
@@ -40,6 +40,8 @@ const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)
function inlineSkill(): string { function inlineSkill(): string {
return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, { return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, {
// LIVE-REPO CWD: git show needs this repo's history to read the
// pre-Phase-1 SKILL.md render at INLINE_REF.
cwd: ROOT, cwd: ROOT,
encoding: 'utf-8', encoding: 'utf-8',
maxBuffer: 8 * 1024 * 1024, maxBuffer: 8 * 1024 * 1024,