gstack/test/skill-e2e-plan-ceo-review-section-loading.test.ts

/**
 * /plan-ceo-review section-loading E2E (periodic, paid, SDK capture) — v2 plan
 * Phase B carve backstop. The per-PR guard is the free static test
 * skill-ceo-section-ordering.test.ts; THIS is the behavioral proof that a real
 * agent actually Reads the carved section instead of working from memory.
 *
 * Detection is LOSSLESS. Earlier this test drove a real PTY and scraped the ANSI
 * screen buffer for the `sections/<file>.md` path. That silently saw nothing in a
 * Conductor PTY — cursor-positioned tool renders and an unanswered Step 0 question
 * loop both defeat the regex, so it reported `read: []` even when the agent did the
 * work. It now runs the skill through `claude -p` (the SDK path the AUQ matrix
 * uses) and detects section reads from the tool-use stream (`Read` calls whose
 * file_path contains `sections/review-sections.md`, or Bash prints whose output
 * contains every line of that section). No rendering layer to mangle.
 *
 * Hermetic, not install-mutating: the freshly-generated worktree skeleton +
 * sections are copied into a throwaway fixture dir and the absolute path is pinned,
 * so the test validates THIS branch's carve without touching the user's active
 * ~/.claude install. (Install-layout linking is covered separately by
 * setup-sections-linking.test.ts.)
 *
 * The agent is told AskUserQuestion is unavailable, so its bounded author
 * policy resolves in-scope choices through Step 0 and the STOP-Read. HOLD
 * SCOPE is the simplest mode that still requires the full review section. Cost:
 * ~$1-2/run. Periodic tier.
 */

import { afterAll, test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
  setupSkillDir,
  skillFromWorktree,
  captureSectionReads,
  validateCeoReviewCompletion,
  hasDisabledOutsideReview,
  LONG_SECTION_CAPTURE_MS,
} from './helpers/auq-sdk-capture';
import { CEO_SECTION_CACHE_PLAN, CEO_SECTION_DECISION_POLICY, hasStaleFillRaceFinding } from './helpers/ceo-section-loading-fixture';
import { createEvalCollector, finalizeEvalCollector, recordE2E } from './helpers/e2e-helpers';

const describeE2E = describeE2ETier('periodic');
const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
const collector = createEvalCollector('e2e-plan-ceo-section-loading');
afterAll(async () => { await finalizeEvalCollector(collector); });

// Sections every plan-ceo-review run must consult after Step 0.
const REQUIRED_SECTIONS = ['review-sections.md'];

describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () => {
  test(
    'a real review Reads the carved section before producing the report',
    async () => {
      const { skillMd, sectionsFrom } = skillFromWorktree('plan-ceo-review');
      const planDir = setupSkillDir({
        skillName: 'plan-ceo-review',
        skillMd,
        sectionsFrom,
        fixtures: { 'PLAN.md': CEO_SECTION_CACHE_PLAN },
        tmpPrefix: 'gstack-ceo-secload-',
      });

      const capture = await captureSectionReads({
        planDir,
        skillName: 'plan-ceo-review',
        scenario:
          'Review the plan in PLAN.md. Hold the current scope (HOLD SCOPE mode) — do not challenge or expand scope. Run the full CEO review. Treat the explicitly accepted repository, adapter and controller contracts as fixture facts; an unavailable implementation is not evidence that those contracts fail. Propose remedies for demonstrated gaps, and surface any actual contradiction without silently weakening a retained requirement. PLAN.md is both the active plan and final output: preserve and amend its plan content, then include the full review report there.',
        decisionPolicy: CEO_SECTION_DECISION_POLICY,
        // The skill appends its report to the active plan. Use that same
        // artifact so the capture does not request a second report write.
        reportFile: 'PLAN.md',
        requiredSections: REQUIRED_SECTIONS,
        reportMarker: /^## GSTACK REVIEW REPORT\s*$/m,
        testName: 'plan-ceo-section-loading',
        runId,
        // This external carve missed the generic loader's v1.71 budget fix:
        // a full 11-section review needs its long work budget, not the helper's
        // ordinary 300s default. The outer CAPTURE_LONG_MS remains unchanged.
        timeout: LONG_SECTION_CAPTURE_MS,
        // This case measures native section loading and report completion;
        // outside-provider dispatch is covered by the cross-harness evals.
        nativeReviewOnly: true,
      });

      try { assertSectionLoadingReport(capture); }
      catch (error) {
        recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result,
          { passed: false, error: String(error), output: capture.output });
        throw error;
      }
      recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result,
        { passed: true, output: capture.output });
    },
    CAPTURE_LONG_MS,
  );
});

function assertSectionLoadingReport(capture: Awaited<ReturnType<typeof captureSectionReads>>): void {
  validateCeoReviewCompletion(capture);
  const { readSections, reportProduced, output } = capture;
  const missing = REQUIRED_SECTIONS.filter(s => !readSections.has(s));
  expect({ reportProduced, read: [...readSections], missing }).toEqual({
    reportProduced: true,
    read: expect.any(Array),
    missing: [],
  });
  // Guard against an empty pass: the report must have real content.
  expect(output.trim().length).toBeGreaterThan(200);
  expect(output).toMatch(/^\|\s*Review\s*\|\s*Trigger\s*\|\s*Why\s*\|\s*Runs\s*\|\s*Status\s*\|\s*Findings\s*\|/m);
  expect(output).toMatch(/^\|\s*CEO Review\s*\|/m);
  // A native capture must not invent an outside dispatch or claim coverage.
  expect(hasDisabledOutsideReview(output)).toBe(true);
  // Loading a section and producing a table alone must not hide an empty
  // review: the complete fixture still contains a real ordering defect.
  expect(hasStaleFillRaceFinding(output)).toBe(true);
}