gstack/test/helpers/auq-sdk-capture.ts

/**
 * SDK-based AUQ capture — the reliable way to grade AskUserQuestion content.
 *
 * Real-PTY capture is lossy for plan-mode AUQs: they render every option on one
 * cursor-positioned logical line that stripAnsi can't reconstruct, so format
 * predicates (ELI10:, Net:, ✅) silently miss even when the question is
 * well-formed. This helper instead uses the `claude -p` SDK path (the same one
 * skill-e2e-plan-format uses): the agent is told to WRITE the verbatim text of
 * the AskUserQuestion it would have asked to a file. That captures exactly what
 * the model GENERATES — the surface where carving could degrade quality — with
 * zero rendering loss. The TTY rendering layer is identical for fat and slim
 * skills, so it is not where token-reduction degradation can hide.
 */
import { resolveEvalModel } from '../../lib/eval-model';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { runSkillTest, type SkillTestResult } from './session-runner';

const ROOT = path.resolve(__dirname, '..', '..');

/**
 * Existing long section-loader work budget (v1.71): complete workflows can
 * load their sections quickly, then need 300–450s to generate the full report.
 * Keep 120s of the CAPTURE_LONG_MS outer budget for setup and reporting.
 * Ordinary captureSectionReads callers retain the 300s default below.
 */
export const LONG_SECTION_CAPTURE_MS = 480_000;

/** The 7 decision-brief format elements graded on the captured AUQ text. */
export const AUQ_FORMAT_ELEMENTS: Array<{ field: string; re: RegExp }> = [
  { field: 'ELI10:', re: /ELI10\s*:/i },
  { field: 'Recommendation:', re: /Recommendation\s*:/i },
  { field: 'Pros / cons:', re: /Pros\s*\/\s*cons/i },
  { field: '✅', re: /✅/ },
  { field: '❌', re: /❌/ },
  { field: 'Net:', re: /Net\s*:/i },
  { field: '(recommended)', re: /\(recommended\)/i },
];

export function scoreAuqFormat(text: string): { present: number; total: number; missing: string[] } {
  const missing = AUQ_FORMAT_ELEMENTS.filter(e => !e.re.test(text)).map(e => e.field);
  return { present: AUQ_FORMAT_ELEMENTS.length - missing.length, total: AUQ_FORMAT_ELEMENTS.length, missing };
}

/**
 * Grade recommendation substance ROBUST to the connective. judgeRecommendation()
 * keys on the literal "because" (correct for the spec, pinned by
 * llm-judge-recommendation.test.ts), but skills routinely write equally
 * substantive reasons as "Recommendation: A. <reason>" / "A — <reason>" /
 * "A: <reason>". Grading those as substance-1 would make the matrix cry wolf on
 * genuinely good recommendations. So we normalize a non-"because" connective to
 * "because" purely for grading, then call the shared judge. We also report
 * whether the ORIGINAL used the literal "because" — a soft style signal, since
 * the format spec prefers it and the voice rule forbids the em-dash form.
 *
 * This does NOT touch judgeRecommendation or its pinned fixtures.
 */
export async function gradeAuqRecommendation(
  text: string,
): Promise<{ substance: number; present: boolean; hadLiteralBecause: boolean; reason: string }> {
  const { judgeRecommendation } = await import('./llm-judge');
  const recLine = text.match(/^[*_]*\s*recommendation\s*[*_]*\s*:\s*(.+)$/im);
  const hadLiteralBecause = !!recLine && /\bbecause\s+\S/i.test(recLine[1]);

  let graded = text;
  if (recLine && !hadLiteralBecause) {
    // Rewrite "Recommendation: <choice><sep><reason>" → "...<choice> because <reason>"
    // sep ∈ {". ", " — ", " - ", ": "} right after a short choice token.
    const normalizedLine = recLine[1].replace(
      /^([^.:—-]{1,40}?)\s*(?:\.\s+|\s*[—-]\s+|:\s+)(\S.+)$/,
      '$1 because $2',
    );
    if (normalizedLine !== recLine[1]) {
      graded = text.replace(recLine[0], `Recommendation: ${normalizedLine}`);
    }
  }

  try {
    const r = await judgeRecommendation(graded);
    return { substance: r.reason_substance, present: r.present, hadLiteralBecause, reason: r.reason_text };
  } catch {
    return { substance: 0, present: !!recLine, hadLiteralBecause, reason: '' };
  }
}

/**
 * Build a throwaway plan dir holding a SPECIFIC plan-ceo-review SKILL.md (so we
 * can pit the carved skeleton against the verbose monolith). `sectionsFrom`, if
 * given, copies that dir's sections/ alongside (for the carved variant).
 */
export function setupPlanCeoDir(opts: {
  skillMd: string;
  sectionsFrom?: string | null;
  tmpPrefix?: string;
}): string {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), opts.tmpPrefix ?? 'auq-sdk-'));
  const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 });
  run('git', ['init', '-b', 'main']);
  run('git', ['config', 'user.email', 'test@test.com']);
  run('git', ['config', 'user.name', 'Test']);
  fs.writeFileSync(
    path.join(dir, 'plan.md'),
    [
      '# Plan: Launch a "developer-friendly" pricing tier',
      '',
      '## Goal',
      'Increase developer adoption.',
      '',
      '## Success metric',
      'More signups.',
      '',
      '## Premise',
      "We haven't talked to any developers about whether the current pricing is a",
      'barrier. The team agreed it "feels like" it should be cheaper.',
    ].join('\n'),
  );
  fs.mkdirSync(path.join(dir, 'plan-ceo-review'), { recursive: true });
  fs.writeFileSync(path.join(dir, 'plan-ceo-review', 'SKILL.md'), opts.skillMd);
  if (opts.sectionsFrom && fs.existsSync(opts.sectionsFrom)) {
    fs.cpSync(opts.sectionsFrom, path.join(dir, 'plan-ceo-review', 'sections'), { recursive: true });
  }
  run('git', ['add', '.']);
  run('git', ['commit', '-m', 'plan']);
  return dir;
}

/**
 * Generic: build a throwaway dir holding ANY skill's SKILL.md (+ optional
 * sections) plus arbitrary fixture files, so the matrix can drive each skill to
 * its first AUQ. Mirrors setupPlanCeoDir but skill-agnostic.
 */
export function setupSkillDir(opts: {
  skillName: string;
  skillMd: string;
  sectionsFrom?: string | null;
  fixtures?: Record<string, string>;
  tmpPrefix?: string;
}): string {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), opts.tmpPrefix ?? `auq-${opts.skillName}-`));
  const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 });
  run('git', ['init', '-b', 'main']);
  run('git', ['config', 'user.email', 'test@test.com']);
  run('git', ['config', 'user.name', 'Test']);
  for (const [name, content] of Object.entries(opts.fixtures ?? {})) {
    const p = path.join(dir, name);
    fs.mkdirSync(path.dirname(p), { recursive: true });
    fs.writeFileSync(p, content);
  }
  fs.mkdirSync(path.join(dir, opts.skillName), { recursive: true });
  fs.writeFileSync(path.join(dir, opts.skillName, 'SKILL.md'), opts.skillMd);
  if (opts.sectionsFrom && fs.existsSync(opts.sectionsFrom)) {
    fs.cpSync(opts.sectionsFrom, path.join(dir, opts.skillName, 'sections'), { recursive: true });
  }
  run('git', ['add', '.']);
  run('git', ['commit', '-m', 'fixture']);
  return dir;
}

/** Read any skill's current (worktree) SKILL.md + its sections dir if present. */
export function skillFromWorktree(skillName: string): { skillMd: string; sectionsFrom: string | null } {
  const sec = path.join(ROOT, skillName, 'sections');
  return {
    skillMd: fs.readFileSync(path.join(ROOT, skillName, 'SKILL.md'), 'utf-8'),
    sectionsFrom: fs.existsSync(sec) ? sec : null,
  };
}

/**
 * Generic: drive ANY skill to its FIRST AskUserQuestion and capture the
 * verbatim decision-brief text the model would have shown. `scenario` is the
 * per-skill prose that triggers a real AUQ (e.g. "review plan.md", "audit
 * vuln.ts for security"). Absolute skill path + Read/Write-only so the agent
 * cannot wander to the global install.
 */
export async function captureFirstAuq(opts: {
  planDir: string;
  skillName: string;
  scenario: string;
  testName: string;
  runId?: string;
  model?: string;
}): Promise<string> {
  const outFile = path.join(opts.planDir, 'ask-capture.md');
  const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
  const prompt = `You are running a format-capture test. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT search for, Glob, find, or read any other SKILL.md anywhere — especially nothing under ~/.claude or /Users.

Read ${skillPath} and follow its workflow for this scenario:

${opts.scenario}

This is a capture test, not an interactive session. Skip any system-audit / environment-setup / codebase-exploration steps. When you reach the FIRST point where the skill would call AskUserQuestion, write the verbatim full decision-brief text of that question (title, ELI10, stakes, recommendation, every option with its ✅/❌ pros/cons bullets, and the Net line) to ${outFile}. Do NOT call any tool to ask the user. Do NOT paraphrase. After writing the file, STOP.`;

  await runSkillTest({
    prompt,
    workingDirectory: opts.planDir,
    allowedTools: ['Read', 'Write'],
    maxTurns: 14,
    timeout: 240_000,
    testName: opts.testName,
    runId: opts.runId,
    model: resolveEvalModel('capture', opts.model),
  });

  try {
    return fs.readFileSync(outFile, 'utf-8');
  } catch {
    return '';
  }
}

/**
 * Drive ANY carved skill through a real `claude -p` run and detect, LOSSLESSLY,
 * which `sections/<file>.md` files the agent actually Read — from the tool-use
 * stream, not the ANSI screen buffer. This is the reliable replacement for the
 * real-PTY `visibleSince()` screen-scraping the section-loading tests used to do
 * (which silently saw nothing in a Conductor PTY: cursor-positioned renders and
 * an unanswered Step 0 question loop both defeat the regex).
 *
 * The skill under test is the planted copy in `planDir` (pin the absolute path so
 * the agent cannot wander to the global install). AskUserQuestion is declared
 * unavailable so the agent auto-picks the recommended option and proceeds far
 * enough to hit the post-Step-0 STOP-Read directives; Read is the tool a STOP-Read
 * resolves to, so Read/Grep/Glob/Write/Edit cover local artifacts (no Bash → it cannot
 * `find /` its way out, nor run git/gh mutations).
 */
export function hasDisabledOutsideReview(output: string): boolean {
  const headings = [...output.matchAll(/^## GSTACK REVIEW REPORT[ \t]*\r?$/gm)];
  const heading = headings.at(-1);
  if (!heading) return false;
  const section = output.slice(heading.index! + heading[0].length).split(/^##[ \t]+/m, 1)[0];
  const plain = (cell: string) => cell.replace(/[*_`]/g, '').trim().replace(/\s+/g, ' ').toLowerCase();
  for (const line of section.split('\n')) {
    if (!line.trimStart().startsWith('|')) continue;
    const cells = line.split('|').map(plain);
    if (cells[1] === 'outside review') {
      return /^disabled(?:$|\s|[(:—–-])/.test(cells[5] ?? '');
    }
  }
  return false;
}

export async function captureSectionReads(opts: {
  planDir: string;
  skillName: string;
  /** Fixture-authorized local artifact commands. */
  artifactCommands?: string;
  scenario: string;
  /** The fixture actor's decision authority; defaults to recommended choices. */
  decisionPolicy?: string;
  /** Relative filename the agent writes its final output to (terminal signal). */
  reportFile?: string;
  /** Marker proving a real report/plan was produced (default: any non-empty text). */
  reportMarker?: RegExp;
  testName: string;
  runId?: string;
  model?: string;
  maxTurns?: number;
  timeout?: number;
  /** Measure native section loading with the documented extra-review opt-out. */
  nativeReviewOnly?: boolean;
}): Promise<{ readSections: Set<string>; reportProduced: boolean; reportWritten: boolean;
  exitReason: SkillTestResult['exitReason']; toolCalls: SkillTestResult['toolCalls'];
  transcript: SkillTestResult['transcript']; output: string }> {
  const outFile = path.join(opts.planDir, opts.reportFile ?? 'REPORT.md');
  const timeout = opts.timeout ?? 300_000;
  const fullPlanReview = opts.skillName === 'plan-ceo-review' || opts.skillName === 'plan-eng-review';
  // The Eng actor may run local review writers. Reject foreign destinations
  // before creating its state or starting a child, including symlink escapes.
  // Tool approval is not a filesystem sandbox; this checks fixture ownership.
  if (opts.skillName === 'plan-eng-review' && opts.nativeReviewOnly && opts.artifactCommands) {
    if (fs.lstatSync(opts.planDir).isSymbolicLink()) throw new Error('Eng section fixture root must not be a symlink');
    const owner = fs.realpathSync(opts.planDir);
    const isInside = (root: string, target: string) => {
      const relative = path.relative(root, target);
      return relative !== '..' && !relative.startsWith('..' + path.sep) && !path.isAbsolute(relative);
    };
    if (path.resolve(outFile) === path.resolve(opts.planDir) || !isInside(path.resolve(opts.planDir), path.resolve(outFile))) {
      throw new Error('Eng section report must stay inside its fixture root');
    }
    let existing = path.resolve(outFile);
    for (;;) {
      try { fs.lstatSync(existing); break; }
      catch (error) {
        if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
        existing = path.dirname(existing);
      }
    }
    if (!isInside(owner, fs.realpathSync(existing))) throw new Error('Eng section report must stay inside its fixture root');
  }
  const readReport = (): Buffer | undefined => {
    try { return fs.readFileSync(outFile); }
    catch (error) {
      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
      return undefined;
    }
  };
  const beforeReport = readReport();
  const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
  // Outside-review dispatch has separate behavioral coverage. Native-only
  // captures use the real supported control in state owned by this call;
  // never mutate the operator's or another capture's gstack configuration.
  // Keep the model-facing config path relative to the fixture's working directory.
  const stateDir = opts.nativeReviewOnly
    ? fs.mkdtempSync(path.join(path.resolve(opts.planDir), '.gstack-section-state-')) : null;
  const nativeReviewRule = stateDir
    ? `\n- Read ${path.relative(path.resolve(opts.planDir), path.join(stateDir, 'config.yaml'))}, the isolated gstack configuration for this capture. It sets codex_reviews: disabled. Follow that documented control: skip the entire extra outside-review step, including its native fallback, and report outside coverage as disabled. Complete all native review sections and the full required report.`
    : '';
  // The actor needs the same execution window as the runner to allocate work.
  // Pacing does not change the review's required content or completion gates.
  const planReviewWritingRule = fullPlanReview
    ? `\n- Native execution window: ${timeout / 1000} seconds. Use the system notice's runner-bound deadline and the observed clock to reserve the final ${timeout / 4000} seconds for assembling and verifying completion outputs. Bash may additionally run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\` for a read-only clock check, including when other Bash commands are restricted. Every required section, finding, approval and output still has to be completed; this is pacing guidance, not permission to skip work.
1. Read the required skill and source material, then resolve Step 0 under the supplied author policy. ${opts.skillName === 'plan-eng-review' ? 'When Scope Challenge finishes, save its outcome. Check the Write/Edit result and Read the saved outcome before advancing.' : 'Follow each workflow save/readback checkpoint as it occurs; do not defer all persistence to one final Write.'} Preserve original requirements and accepted plan amendments throughout.
2. Whenever Step 0 or a section needs an independent choice, give it one ID and one authoritative decision record; findings may reference several choice IDs. Before auto-selecting, save its full currentDecision question/header, every labeled option and full description, commitment comparison and source citations. Read back and verify those fields against the drafted decision and cited source; fix mismatches first. Then record the authorized auto-decision and exact scope, apply answers and amendments with scoped Edit operations, and Read back before taking another row. Keep the complete question and every option before selecting, as the skill requires.
3. Work through ${opts.skillName === 'plan-ceo-review' ? 'all 11 sections, giving each an explicit outcome (including no issues or justified skips)' : 'all four review sections, including an explicit "No issues found" when applicable'}. ${opts.skillName === 'plan-eng-review' ? 'After each completed review section, save its complete findings and disposition with concrete evidence, the selected remedy, residual risks, and verification. Check the Write/Edit result and Read the saved outcome before advancing to the next section; these checkpoints also apply when no new choice needs approval.' : 'As sections finish, add each finding once with concrete evidence, the selected remedy, residual risks, and verification.'} Use compact outcomes, short option bullets and table cells. Execute checklists without copying their questions or narrating every check. Cross-reference saved IDs instead of repeating findings, option deliberations, diagrams or registries; do not regenerate unchanged records or repeat their briefs in the final report. Do not write full implementation or test code unless needed to specify an accepted change.
4. Use the completion reserve to assemble ${opts.skillName === 'plan-ceo-review' ? 'the complete required registries, applicable diagrams, tasks, completion summary, and exact GSTACK REVIEW REPORT table' : 'the required diagrams, test-plan artifact, tasks, TODOS dispositions, completion summary and GSTACK REVIEW REPORT'}. Preserve every finding and original requirement, required decision fields and comparisons, exact approvals and verification; every diagram must retain its specified format. Read back the assembled plan and verify the full required outputs. Finish missing required outputs with scoped Edits, without rewriting unchanged records, then perform the skill's full final Read-back gate. Complete every required artifact and verification before returning.`
    : '';
  const prompt = `You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users.

Read ${skillPath} and EXECUTE its workflow for this scenario:

${opts.scenario}

Rules for this run:
- Skip system-audit, environment-setup, telemetry, and unrelated codebase exploration. Read the supplied plan's referenced fixture files when its review requires them.
${opts.decisionPolicy ?? "- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask."}
- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/<file>\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory.
- Resolve installed-root paths for section and companion Markdown files under ${opts.planDir}, where this fixture's skill package is copied.
- Do NOT run git, gh, commit, push, or any mutating command${opts.artifactCommands ? ' except the local artifact commands explicitly authorized below' : ''}.${opts.artifactCommands ? `\n- ${opts.artifactCommands}` : ''}
${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} with Write/Edit at the workflow checkpoints below.` : `- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${outFile}.`}${nativeReviewRule}${planReviewWritingRule}
- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`;

  let result: SkillTestResult;
  try {
    if (stateDir) fs.writeFileSync(path.join(stateDir, 'config.yaml'), 'codex_reviews: disabled\n');
    result = await runSkillTest({
      prompt,
      workingDirectory: opts.planDir,
      allowedTools: ['Read', 'Grep', 'Glob', 'Write', 'Edit', ...(opts.nativeReviewOnly ? [] : ['Agent']), ...(opts.artifactCommands || fullPlanReview ? ['Bash'] : [])],
      tools: ['Read', 'Grep', 'Glob', 'Write', 'Edit', ...(opts.nativeReviewOnly ? [] : ['Agent']), ...(opts.artifactCommands || fullPlanReview ? ['Bash'] : [])],
      publicStreamDiagnostics: true,
      ...(fullPlanReview ? { completionReserveMs: timeout / 4 } : {}),
      maxTurns: opts.maxTurns ?? 25,
      timeout,
      testName: opts.testName,
      runId: opts.runId,
      model: resolveEvalModel('capture', opts.model),
      ...(stateDir ? { env: { GSTACK_HOME: stateDir, GSTACK_STATE_ROOT: stateDir } } : {}),
    });
  } finally {
    if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true });
  }

  const readSections = new Set<string>();
  for (const c of result.toolCalls) {
    if (c.tool !== 'Read') continue;
    const fp = String(c.input?.file_path ?? '');
    const m = fp.match(/(?:^|[\\/])sections[\\/]([A-Za-z0-9._-]+\.md)(?=$|[?#])/);
    if (m) readSections.add(m[1]);
  }

  const afterReport = readReport();
  const reportWritten = afterReport !== undefined
    && (beforeReport === undefined || !afterReport.equals(beforeReport));
  // An unchanged seed (including a same-byte rewrite) is not this attempt's report.
  const output = reportWritten ? afterReport!.toString('utf-8') : result.output ?? '';
  const reportProduced = result.exitReason === 'success'
    && (opts.reportMarker ? opts.reportMarker.test(output) : output.trim().length > 0);

  // Keep successful terminal-output captures, but a draft left by a failed run
  // must never satisfy callers that use reportProduced as their completion gate.
  return { readSections, reportProduced, reportWritten, exitReason: result.exitReason, toolCalls: result.toolCalls, transcript: result.transcript, output };
}

/** A completed CEO review needs its artifact and every summary outcome. */
export function validateCeoReviewCompletion(capture: {
  exitReason: string;
  reportWritten: boolean;
  output: string;
}): void {
  if (capture.exitReason !== 'success') {
    throw new Error(`CEO review execution failed: ${capture.exitReason}`);
  }
  if (!capture.reportWritten) throw new Error('CEO review did not write REPORT.md');
  const lines = capture.output.split(/\r?\n/);
  const summaryStart = lines.findIndex(line =>
    /^(?:#{1,6}\s+(?:\d+[.)]\s+)?)?(?:\|\s*)?(?:MEGA PLAN REVIEW\s*[—–-]\s*)?COMPLETION SUMMARY(?:\s*\|)?$/i
      .test(line.trim().replace(/\*\*/g, '')),
  );
  if (summaryStart === -1) throw new Error('CEO report is missing its Completion Summary');
  const summaryLines = lines.slice(summaryStart + 1);
  const nextHeading = summaryLines.findIndex(line => /^#{1,6}\s+\S/.test(line));
  const summary = nextHeading === -1 ? summaryLines : summaryLines.slice(0, nextHeading);
  const outcomes = new Map<number, string>();
  for (const line of summary) {
    const row = line.replace(/\*\*/g, '').match(/^\s*\|\s*Section\s+(\d{1,2})\b[^|]*\|\s*(.*?)\s*\|\s*$/i);
    if (row) outcomes.set(Number(row[1]), row[2].replace(/\*\*/g, '').trim());
  }
  for (let section = 1; section <= 11; section++) {
    const outcome = outcomes.get(section) ?? '';
    const placeholder = !outcome || /___/.test(outcome)
      || /^(?:TBD|TODO|pending|not reviewed|done|complete(?:d)?|reviewed|[-—]+)[.!]?$/i.test(outcome);
    const skipped = /^(?:skip(?:ped)?|N\/A|not applicable)\b/i.test(outcome);
    const noUi = section === 11 && /\bno UI\b/i.test(outcome);
    if (placeholder || (skipped && !noUi)) {
      throw new Error(`CEO Completion Summary is missing a completed Section ${section} outcome`);
    }
  }
}

/** Read the carved (current worktree) plan-ceo SKILL.md + its sections dir. */
export function carvedSkill(): { skillMd: string; sectionsFrom: string | null } {
  const sec = path.join(ROOT, 'plan-ceo-review', 'sections');
  return {
    skillMd: fs.readFileSync(path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), 'utf-8'),
    sectionsFrom: fs.existsSync(sec) ? sec : null,
  };
}

/** Read the pre-carve verbose monolith plan-ceo SKILL.md.
 *  VENDORED fixture (v1.75 precedent), not a git ref: the old default
 *  `git show ab66193e^:...` pinned a BRANCH-LOCAL commit — it dies the day
 *  that branch is pruned and already fails on shallow clones. The fixture
 *  is the frozen pre-cut render; test/git-ref-fixture-tripwire.test.ts
 *  keeps this class from coming back. */
export function verboseSkill(): string {
  return fs.readFileSync(
    path.join(ROOT, 'test', 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md'),
    'utf-8',
  );
}

function execGit(args: string[]): string {
  const r = spawnSync('git', args, { cwd: ROOT, encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024, timeout: 30_000 });
  if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${r.stderr}`);
  return r.stdout;
}

/**
 * Drive plan-ceo-review to its Step 0F mode-selection AskUserQuestion in the
 * given plan dir and capture the verbatim question text the model generates.
 * Returns the captured text ('' if the agent never wrote the file).
 */
export async function captureModeSelectionAuq(opts: {
  planDir: string;
  testName: string;
  runId?: string;
  model?: string;
}): Promise<string> {
  const outFile = path.join(opts.planDir, 'ask-capture.md');
  const skillPath = path.join(opts.planDir, 'plan-ceo-review', 'SKILL.md');
  const planPath = path.join(opts.planDir, 'plan.md');
  // CRITICAL: pin the EXACT skill file. Without this the agent runs
  // `find / -name SKILL.md` / Glob and reads the GLOBAL install
  // (~/.claude/skills/...) instead of the version-under-test in the temp dir —
  // which silently invalidates a carved-vs-verbose A/B (both sides end up
  // reading the same global skill). Absolute path + no-wander instruction +
  // Bash disallowed (so `find /` is impossible) locks it to the planted file.
  const prompt = `You are running a format-capture test. Use ONLY these two files:
  - The skill to follow: ${skillPath}
  - The plan to review: ${planPath}

Read ${skillPath} for the review workflow. Do NOT search for, Glob, find, or read any OTHER SKILL.md anywhere on the system — especially nothing under ~/.claude or /Users. The ONLY skill file you may read is the absolute path above.

Read ${planPath} — that is the plan to review. It is a standalone plan document, not a codebase. Skip any codebase exploration or system-audit steps.

Proceed to Mode Selection, where the skill presents the 4 review-mode options to the user via AskUserQuestion.

Write the verbatim text of that AskUserQuestion (the full decision brief: title, ELI10, stakes, recommendation, every option with its pros/cons bullets, and the Net line) to ${outFile}. Do NOT call any tool to ask the user. Do NOT paraphrase. After writing the file, stop.`;

  await runSkillTest({
    prompt,
    workingDirectory: opts.planDir,
    // Read + Write only: no Bash means the agent cannot `find /` its way to the
    // global install, and the skill's preamble bash blocks (irrelevant to format
    // capture) can't run and wander.
    allowedTools: ['Read', 'Write'],
    maxTurns: 12,
    timeout: 240_000,
    testName: opts.testName,
    runId: opts.runId,
    model: resolveEvalModel('capture', opts.model),
  });

  try {
    const text = fs.readFileSync(outFile, 'utf-8');
    // Defense in depth: verify the agent actually read the planted skill, not a
    // global one. If the captured run somehow read elsewhere we can't detect it
    // from the output file alone, so callers should also confirm via the run
    // log; this guard at least catches an empty/placeholder capture.
    return text;
  } catch {
    return '';
  }
}