gstack/test/skill-e2e-plan-eng-artifact.test.ts

/**
 * /plan-eng-review writes its QA test-plan artifact when its live Test review
 * runs (periodic checkpoint, paid, real-PTY).
 *
 * Replaces the composing `plan-eng-review-artifact` case in skill-e2e-plan.test.ts.
 * That case told a `claude -p` session to skip the preamble and questions and
 * "go straight to the review"; in runs 37151477069, 37158847998 and 37162480720
 * (and one local CI-image run) the model wrote the whole review into one file
 * without executing the workflow, so it measured composing, not the skill. A
 * full interactive run does write the artifact (37158847998's batching PTY run
 * wrote eng-review-test-plan-20261003-224617.md during Test review).
 *
 * A fresh run spends its budget on startup and one-at-a-time Scope Challenge
 * questions (census 37174266054: six questions, timeout at 595 s, no Test
 * review). This checkpoint resumes the real interactive /plan-eng-review from
 * the record a real run saves: the plan, Scope Challenge result, Sections 1-2
 * dispositions and a Decision ledger rebuilt from that census run's six actual
 * native questions and answers. The live skill then runs Section 3, Test
 * review, answering each question with its first option, and the case stops
 * once the artifact exists or the report completes. The requirement is
 * unchanged: exactly one new QA test plan under the run's own state root, about
 * the reviewed dashboard change. (The fresh start-to-finish marathon case
 * retired with the standalone skill, GST-1698.)
 */

import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { DASHBOARD_PLAN, SOURCES } from './helpers/plan-eng-artifact-fixture';
import { engResumeProblems, engResumeReport, type EngResumeCall } from './helpers/plan-eng-resume';
import resume from './fixtures/plan-eng-artifact-resume-37174266054.json';
import { describeE2ETier } from './helpers/e2e-gate';
import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, hasCompletePlanReport, runPlanSkillCounting } from './helpers/claude-pty-runner';

const describeE2E = describeE2ETier('periodic');

describeE2E('/plan-eng-review QA test-plan artifact (periodic)', () => {
  test('an interactive review writes one QA test plan about the reviewed change', async () => {
    const startedAt = Date.now();
    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-artifact-'));
    const planPath = path.join(tmpDir, 'gstack-test-plan-eng-artifact.md');
    const calls = resume.calls as EngResumeCall[];
    const plan = DASHBOARD_PLAN.slice(DASHBOARD_PLAN.indexOf('# Plan: Add Dashboard'));
    const report = engResumeReport(plan, calls);
    expect(engResumeProblems(report, calls)).toEqual([]);
    fs.writeFileSync(planPath, report);
    try {
      const obs = await runPlanSkillCounting({
        skillName: 'plan-eng-review',
        slashCommand: '/plan-eng-review',
        followUpPrompt: [
          'Proceed directly to the requested engineering review; skip the optional /office-hours prerequisite.',
          `Resume my /plan-eng-review of this plan. Its report file is ${planPath}: it holds the plan, the completed Scope Challenge (D1-D${calls.length} answered) and Sections 1-2 (Architecture, Code quality) with no open choices, plus the Decision ledger. Read it first, keep using it as the report file, and do not redo or re-ask those stages. Continue at Section 3, Test review, and carry the review on from there.`,
          '',
          plan,
        ].join('\n'),
        fixtureFiles: SOURCES,
        permissionPlanPath: planPath,
        isLastStep0AUQ: engStep0Boundary,
        isSetupAUQ: engSetupAUQ,
        isFirstReviewAUQ: engFirstReviewAUQ,
        // The artifact is the only verdict; stop once it exists or the report is complete.
        isCollectionComplete: (_transcript, _fingerprints, testPlans) =>
          testPlans.length > 0 || hasCompletePlanReport(planPath, startedAt, Date.now()),
        reviewCountCeiling: Infinity,
        preconfiguredReviewActor: true,
        timeoutMs: CAPTURE_LONG_MS,
        env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
      });
      const plans = obs.engTestPlans;
      console.log(`plan-eng-review-artifact: outcome=${obs.outcome} elapsed=${obs.elapsedMs}ms review=${obs.reviewCount} test plans=${plans.map(plan => plan.file).join(', ') || 'none'}`);
      if (!['plan_ready', 'completion_summary', 'collection_complete'].includes(obs.outcome)) {
        throw new Error(`plan-eng-review-artifact: outcome=${obs.outcome} (${obs.summary})\n--- evidence ---\n${obs.evidence}`);
      }
      expect(plans, `expected one new eng-review test plan; outcome=${obs.outcome}`).toHaveLength(1);
      expect(/dashboard|fetchStats|\/api\/stats/i.test(plans[0]!.content), 'the test plan covers the reviewed dashboard change').toBe(true);
    } finally {
      fs.rmSync(tmpDir, { recursive: true, force: true });
    }
  }, CAPTURE_LONG_MS + 30_000);
});