/** * Plan-mode-info no-op regression (gate tier, paid, real-PTY). * * Asserts: when a plan-review skill is invoked OUTSIDE plan mode (no * --permission-mode plan flag, no plan-mode reminder injected), the skill * still reaches a terminal outcome ('asked' or 'plan_ready'). This is the * negative coverage to the per-skill plan-mode smokes — if plan-mode-keyed * behavior ever starts misfiring for non-plan-mode sessions (e.g., gating * questions on a phrase that isn't there, or the plan-eng/plan-design * scope-gate auto-select-B bypass firing without plan mode), this test * catches it. * * Why this matters: outside plan mode, claude doesn't render a native * confirmation UI. The skill must drive its own AskUserQuestion. Same * runner, same outcome contract — just `inPlanMode: false`. * * Coverage grew with the scope-gate bypass (plan-mode auto-select B): * - plan-ceo-review: original preamble-misfire regression. * - plan-eng-review / plan-design-review: the bypass must NOT fire outside * plan mode (scopeGateAutoSelectObserved stays false), and when the run * ends in 'asked', the question that fired must be the scope gate itself * (outside plan mode with no named target, the gate is the FIRST * question by contract). * - named-target case: a pasted draft (initialPlanContent) IS an * explicitly-named target, so the gate must NOT ask — and the review * must actually consume the pasted content. * * Cost note: 4 sequential PTY runs (~3-5 min each) in the gate lane, up * from 1 pre-bypass. Selected only when plan-ceo/eng/design or the runner * change (see 'plan-mode-no-op' in touchfiles.ts). */ import { describe, test, expect } from 'bun:test'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; const describeE2E = shouldRun ? describe : describe.skip; const PLAN_MODE_REMINDER = 'Plan mode is active. The user indicated that they do not want you to execute yet'; // Distinctive token proves the pasted target was consumed by the review — // not just that no question fired. Nonsense-unique so it can't appear by // coincidence in skill output. const SEED_TOKEN = 'ZephyrLedgerWidget'; const NAMED_TARGET_SEED = ` # Plan: ${SEED_TOKEN} settings panel ## Scope Add a ${SEED_TOKEN} settings panel with a single toggle that enables weekly export emails. One new component, one route, one test file. ## Files - src/components/${SEED_TOKEN}.tsx (new) - src/routes/settings.tsx (add panel) - test/${SEED_TOKEN}.test.tsx (new) `; describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => { for (const skillName of ['plan-ceo-review', 'plan-eng-review', 'plan-design-review'] as const) { test(`${skillName} reaches a terminal outcome outside plan mode`, async () => { const obs = await runPlanSkillObservation({ skillName, inPlanMode: false, timeoutMs: 300_000, // eng/design: force the prose-fallback path. The unconditional // gate-must-ask assert below pins the render shape the detector // anchors on, and only the --disallowedTools prose fallback makes // that shape CONTRACTUAL ("use exactly this shape" in the template); // native AskUserQuestion could render terse option labels that a // correct run would fail on (red-team finding). ...(skillName === 'plan-ceo-review' ? {} : { extraArgs: ['--disallowedTools', 'AskUserQuestion'] }), }); if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { throw new Error( `plan-mode no-op regression FAILED (${skillName}): outcome=${obs.outcome}\n` + `summary: ${obs.summary}\n` + `elapsed: ${obs.elapsedMs}ms\n` + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, ); } expect(['asked', 'plan_ready']).toContain(obs.outcome); // Negative regression: the rendered output must NOT echo the plan-mode // distinctive reminder phrase. If it does, the plan-mode preamble // section is leaking outside plan mode. expect(obs.evidence).not.toContain(PLAN_MODE_REMINDER); if (skillName !== 'plan-ceo-review') { // Scope-gate bypass must not misfire: no auto-select announcement // outside plan mode. expect(obs.scopeGateAutoSelectObserved ?? false).toBe(false); // UNCONDITIONAL: outside plan mode with no named target, the gate is // a hard STOP before any tool call, so the gate question must have // rendered no matter which terminal outcome fired. Gating this on // outcome === 'asked' would let a silent-bypass run that reaches // plan_ready (isPlanReadyVisible also matches common prose) sail // through — the exact regression this test exists to catch. expect(obs.scopeGateQuestionObserved ?? false).toBe(true); } }, 360_000); } // Named-target exception (outside plan mode): a pasted draft IS an // explicitly-named target, so the scope gate must NOT ask — and the // review must consume the pasted content (seed token visible in the // review output), proving the target was used rather than the question // merely skipped. Also the over-trigger guard for the tightened // "explicit-only" exception wording. test('plan-eng-review skips the scope gate for an explicitly-pasted target', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: false, initialPlanContent: NAMED_TARGET_SEED, trackTokens: [SEED_TOKEN], timeoutMs: 300_000, }); if ( obs.outcome === 'wrote_findings_before_asking' || obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout' ) { throw new Error( `named-target no-op FAILED: outcome=${obs.outcome}\n` + `summary: ${obs.summary}\n` + `elapsed: ${obs.elapsedMs}ms\n` + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, ); } expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(obs.evidence).not.toContain(PLAN_MODE_REMINDER); // The pasted doc is the named target: gate question must not render, // no plan-mode announcement either (we are NOT in plan mode). expect(obs.scopeGateQuestionObserved ?? false).toBe(false); expect(obs.scopeGateAutoSelectObserved ?? false).toBe(false); // Target consumption via high-water token tracking over the CUMULATIVE // buffer — the 2KB evidence tail is lossy and the plan-file fallback is // unreachable outside plan mode (extractPlanFilePath only matches // plan-mode save renders). expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true); }, 360_000); });