mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-01 02:40:47 +02:00
fix(test): unique tmp dirs for plan artifacts + audited live-repo cwd sites
Six paid PTY tests wrote their expected plan artifact to a FIXED shared
/tmp path ('/tmp/gstack-test-plan-<mode>.md') and rmSync'd it in
finally — under --retry 1, EVALS_JOBS>1, or two concurrent worktrees, a
sibling's cleanup deletes this run's artifact and the D19 'agent did
not produce expected plan file' assertion fires spuriously. Each test
now mkdtemps its own dir, interpolates the unique path into the agent
prompt (fixture-sourced prompts get a replaceAll + drift guard that
throws if the fixture's literal ever moves), and cleans up its own dir.
The 18 cwd:-into-the-live-repo sites were audited: all deliberate
(skill registry + hermetic pre-trusted dir, in-repo gen renders, git
history reads, slug resolution) — each now carries a
'// LIVE-REPO CWD: <reason>' comment so the next audit can tell
deliberate from accidental.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
0d2f703f28
commit
ae5ffefb9a
@@ -18,6 +18,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
ceoStep0Boundary,
|
||||
@@ -62,8 +64,8 @@ const N_PAIRED = 2;
|
||||
const FLOOR_PAIRED = 2;
|
||||
const CEILING_PAIRED = 4;
|
||||
|
||||
const PLAN_CEO_5_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).',
|
||||
const planCeo5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Payment Processing Integration',
|
||||
'',
|
||||
@@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [
|
||||
'order in a loop.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_CEO_2_PAIRED_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).',
|
||||
const planCeo2PairedFindings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Payment Processing — Test Coverage',
|
||||
'',
|
||||
@@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [
|
||||
'the success path is correctness, the failure path is graceful degradation.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md';
|
||||
const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md';
|
||||
|
||||
describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => {
|
||||
test(
|
||||
`5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: PLAN_CEO_5_FINDINGS,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
|
||||
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir: a hardcoded shared /tmp path collides under
|
||||
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
|
||||
// rmSync deletes this run's artifact → spurious D19 failure).
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: planCeo5Findings(planPath),
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
|
||||
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
|
||||
// D19: review report at bottom of plan file.
|
||||
if (!fs.existsSync(PLAN_CEO_PATH)) {
|
||||
if (!fs.existsSync(planPath)) {
|
||||
throw new Error(
|
||||
`D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` +
|
||||
`D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` +
|
||||
`Either the agent ignored the path instruction in the follow-up prompt, or\n` +
|
||||
`the helper exited before the agent wrote the file. ` +
|
||||
`outcome=${obs.outcome} review=${obs.reviewCount}`,
|
||||
);
|
||||
}
|
||||
const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8');
|
||||
const planContent = fs.readFileSync(planPath, 'utf-8');
|
||||
const verdict = assertReviewReportAtBottom(planContent);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` +
|
||||
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
|
||||
(verdict.trailingHeadings
|
||||
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
|
||||
: '') +
|
||||
@@ -187,7 +188,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
@@ -199,24 +200,24 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
test(
|
||||
`paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_PAIRED + 1,
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir — see the distinct-findings test above.
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: planCeo2PairedFindings(planPath),
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_PAIRED + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`paired-finding control FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -242,7 +243,7 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user