gstack/test/skill-e2e-plan-ceo-plan-mode.test.ts

/**
 * plan-ceo-review plan-mode smoke (gate, paid, real-PTY).
 *
 * Asserts: when /plan-ceo-review is invoked in plan mode, the FIRST terminal
 * outcome is 'asked' — a skill-question numbered list. Permission dialogs
 * (which also render numbered lists) are filtered out by `runPlanSkillObservation`
 * via its `isPermissionDialogVisible(visible.slice(-1500))` short-circuit.
 *
 * Reaching 'plan_ready' first IS the regression we want to catch: the agent
 * skipped Step 0 entirely and went straight to ExitPlanMode. The original
 * failure had the assistant read a diff, write a plan with two issues, and
 * call ExitPlanMode without ever firing AskUserQuestion — the user had to
 * manually call out the missing per-issue questions.
 *
 * Why this skill is special: unlike plan-eng-review / plan-design-review /
 * plan-devex-review (whose smokes accept either 'asked' or 'plan_ready'),
 * plan-ceo-review's template mandates the Step 0 question flow, including
 * Step 0E mode selection, BEFORE proceeding with the review. There is no
 * legitimate path to plan_ready that does not first emit a skill-question
 * numbered prompt.
 *
 * Env passthrough: passes `QUESTION_TUNING=false` and `EXPLAIN_LEVEL=default`
 * via the runner's env option. Today these are advisory — `gstack-config`
 * reads `~/.gstack/config.yaml`, not env vars, so a contributor with
 * `question_tuning: true` set in their YAML config can still see AUTO_DECIDE
 * masking. The env passthrough is wired so a future gstack-config change to
 * honor env overrides will make this test hermetic without further edits.
 * Tracked as a post-merge follow-up.
 *
 * FAIL conditions: 'plan_ready' first, silent Write/Edit before any prompt,
 * claude crash, timeout.
 *
 * See test/helpers/claude-pty-runner.ts for runner internals.
 */

import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { createPlanCountFixture } from './helpers/plan-count-fixture';
import {
  runPlanSkillObservation,
  assertReportAtBottomIfPlanWritten,
} from './helpers/claude-pty-runner';

const describeE2E = describeE2ETier('gate');

// This smoke measures the first review question, so give it a small product
// plan instead of making the checkout that supplies skills its review target.
const PLAN = `# Plan: Archive completed tasks

## Goal
Tiny Tasks is a personal task tracker. Completed tasks stay in the main list,
making it harder to find unfinished work after a busy week. Let users archive
completed tasks without permanently deleting their history.

## Proposed change
- Add an optional archivedAt field to the existing task model in src/tasks.ts.
- Add an Archive completed action; unfinished tasks remain in the active list.
- Add an archive view with a Restore action for accidentally archived tasks.
- Keep older saved tasks compatible when archivedAt is absent.

## Validation
Test mixed active/completed tasks, an empty archive, restoring a task, and
loading existing saved tasks. Pilot with five users and check whether they
can find their next unfinished task more easily.
`;

describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
  test('first terminal outcome is asked (Step 0 fires before any plan write)', async () => {
    const fixture = createPlanCountFixture(PLAN, { files: {
      'README.md': '# Tiny Tasks\n\nA personal task tracker; src/tasks.ts owns the task model and list filtering.\n',
      'src/tasks.ts': 'export type Task = { id: string; title: string; completed: boolean };\n' +
        'export function visibleTasks(tasks: Task[]): Task[] { return tasks; }\n',
    } });
    try {
      const obs = await runPlanSkillObservation({
        skillName: 'plan-ceo-review',
        inPlanMode: true,
        cwd: fixture.cwd,
        // 420s, not 300s: measured 2026-08-11, a clean isolated pass took
        // 295.7s (80s on a quiet main run) — 4s under the old budget — and the
        // same run timed out at ~308s three times under concurrent eval load.
        // Same runner-contention class as review-dashboard-via/retro-base-
        // branch; headroom instead of a budget-edge flake in the gate lane.
        timeoutMs: 420_000,
        env: { ...fixture.env, QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
      });

      if (obs.outcome !== 'asked') {
        const diagnosis =
          obs.outcome === 'plan_ready'
            ? `'plan_ready' first means the agent skipped Step 0 entirely and went straight to ExitPlanMode without asking.`
            : obs.outcome === 'timeout'
              ? `Timeout means the agent neither asked nor completed within the budget — likely hung mid-question or stuck on a permission dialog.`
              : obs.outcome === 'silent_write'
                ? `Silent Write/Edit fired to an unsanctioned path before any AskUserQuestion — also a Step 0 skip.`
                : `Outcome '${obs.outcome}' is unexpected; investigate the evidence below.`;
        throw new Error(
          `plan-ceo-review smoke FAILED: outcome=${obs.outcome}\n` +
            `${diagnosis}\n` +
            `Expected 'asked'. See plan-ceo-review/SKILL.md.tmpl: the Step 0 STOP rules ` +
            `and the "One decision unit = one AskUserQuestion call" rule under "How to ask questions".\n` +
            `summary: ${obs.summary}\n` +
            `elapsed: ${obs.elapsedMs}ms\n` +
            `--- evidence (last 2KB visible) ---\n${obs.evidence}`,
        );
      }
      assertReportAtBottomIfPlanWritten(obs);
    } finally {
      fixture.cleanup();
    }
  }, CAPTURE_LONG_MS);
});