mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-31 18:30:39 +02:00
Garry TanandClaude Fable 5
ae5ffefb9a
Six paid PTY tests wrote their expected plan artifact to a FIXED shared
/tmp path ('/tmp/gstack-test-plan-<mode>.md') and rmSync'd it in
finally — under --retry 1, EVALS_JOBS>1, or two concurrent worktrees, a
sibling's cleanup deletes this run's artifact and the D19 'agent did
not produce expected plan file' assertion fires spuriously. Each test
now mkdtemps its own dir, interpolates the unique path into the agent
prompt (fixture-sourced prompts get a replaceAll + drift guard that
throws if the fixture's literal ever moves), and cleans up its own dir.
The 18 cwd:-into-the-live-repo sites were audited: all deliberate
(skill registry + hermetic pre-trusted dir, in-repo gen renders, git
history reads, slug resolution) — each now carries a
'// LIVE-REPO CWD: <reason>' comment so the next audit can tell
deliberate from accidental.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
55 lines
2.3 KiB
TypeScript
55 lines
2.3 KiB
TypeScript
/**
|
|
* /plan-eng-review AskUserQuestion floor regression (periodic, paid, real-PTY).
|
|
*
|
|
* Catches the May 2026 transcript bug where /plan-eng-review wrote a
|
|
* multi-section review plan to ~/.claude/plans/ and called ExitPlanMode
|
|
* without firing any AskUserQuestion. See
|
|
* `.context/attachments/pasted_text_2026-05-06_10-25-23.txt`.
|
|
*
|
|
* Uses runPlanSkillFloorCheck — a minimal "did the agent fire ANY AUQ?"
|
|
* observer that exits early on the first non-permission numbered-option
|
|
* render. See claude-pty-runner.ts for why this is separate from the
|
|
* runPlanSkillCounting harness used by periodic finding-count tests.
|
|
*
|
|
* Tier: periodic. Budget: 10 min (early exit on success ~30-90s typical).
|
|
* Cost: ~$0.50-$1.50 per run depending on early-exit timing.
|
|
*/
|
|
|
|
import { test } from 'bun:test';
|
|
import { describeE2ETier } from './helpers/e2e-gate';
|
|
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
|
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
|
|
|
|
const describeE2E = describeE2ETier('periodic');
|
|
|
|
describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
|
test(
|
|
'seeded forcing finding causes the agent to fire at least one AskUserQuestion',
|
|
async () => {
|
|
const obs = await runPlanSkillFloorCheck({
|
|
skillName: 'plan-eng-review',
|
|
slashCommand: '/plan-eng-review',
|
|
followUpPrompt: FORCING_FLOOR_ENG,
|
|
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
|
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
|
cwd: process.cwd(),
|
|
timeoutMs: 600_000,
|
|
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
|
});
|
|
|
|
if (obs.outcome !== 'auq_observed') {
|
|
throw new Error(
|
|
`floor test FAILED: outcome=${obs.outcome} elapsed=${obs.elapsedMs}ms\n` +
|
|
`summary: ${obs.summary}\n` +
|
|
`If outcome is plan_ready or completion_summary, this is the transcript-bug ` +
|
|
`regression — agent reached terminal without firing AskUserQuestion. See ` +
|
|
`.context/attachments/pasted_text_2026-05-06_10-25-23.txt.\n` +
|
|
`If outcome is timeout, agent may just be slow — re-run or increase budget.\n` +
|
|
`--- evidence (last 3KB) ---\n${obs.evidence}`,
|
|
);
|
|
}
|
|
},
|
|
660_000,
|
|
);
|
|
});
|