gstack/test/helpers/auq-sdk-capture.ts

/**
 * SDK-based AUQ capture — the reliable way to grade AskUserQuestion content.
 *
 * Real-PTY capture is lossy for plan-mode AUQs: they render every option on one
 * cursor-positioned logical line that stripAnsi can't reconstruct, so format
 * predicates (ELI10:, Net:, ✅) silently miss even when the question is
 * well-formed. This helper instead uses the `claude -p` SDK path (the same one
 * skill-e2e-plan-format uses): the agent is told to WRITE the verbatim text of
 * the AskUserQuestion it would have asked to a file. That captures exactly what
 * the model GENERATES — the surface where carving could degrade quality — with
 * zero rendering loss. The TTY rendering layer is identical for fat and slim
 * skills, so it is not where token-reduction degradation can hide.
 */
import { resolveEvalModel } from '../../lib/eval-model';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { runSkillTest, type SkillTestResult } from './session-runner';

const ROOT = path.resolve(__dirname, '..', '..');

/**
 * Existing long section-loader work budget (v1.71): complete workflows can
 * load their sections quickly, then need 300–450s to generate the full report.
 * Keep 120s of the CAPTURE_LONG_MS outer budget for setup and reporting.
 * Ordinary captureSectionReads callers retain the 300s default below.
 */
export const LONG_SECTION_CAPTURE_MS = 480_000;

/** The 7 decision-brief format elements graded on the captured AUQ text. */
export const AUQ_FORMAT_ELEMENTS: Array<{ field: string; re: RegExp }> = [
  { field: 'ELI10:', re: /ELI10\s*:/i },
  { field: 'Recommendation:', re: /Recommendation\s*:/i },
  { field: 'Pros / cons:', re: /Pros\s*\/\s*cons/i },
  { field: '✅', re: /✅/ },
  { field: '❌', re: /❌/ },
  { field: 'Net:', re: /Net\s*:/i },
  { field: '(recommended)', re: /\(recommended\)/i },
];

export function scoreAuqFormat(text: string): { present: number; total: number; missing: string[] } {
  const missing = AUQ_FORMAT_ELEMENTS.filter(e => !e.re.test(text)).map(e => e.field);
  return { present: AUQ_FORMAT_ELEMENTS.length - missing.length, total: AUQ_FORMAT_ELEMENTS.length, missing };
}

/**
 * Grade recommendation substance ROBUST to the connective. judgeRecommendation()
 * keys on the literal "because" (correct for the spec, pinned by
 * llm-judge-recommendation.test.ts), but skills routinely write equally
 * substantive reasons as "Recommendation: A. <reason>" / "A — <reason>" /
 * "A: <reason>". Grading those as substance-1 would make the matrix cry wolf on
 * genuinely good recommendations. So we normalize a non-"because" connective to
 * "because" purely for grading, then call the shared judge. We also report
 * whether the ORIGINAL used the literal "because" — a soft style signal, since
 * the format spec prefers it and the voice rule forbids the em-dash form.
 *
 * This does NOT touch judgeRecommendation or its pinned fixtures.
 */
export async function gradeAuqRecommendation(
  text: string,
): Promise<{ substance: number; present: boolean; hadLiteralBecause: boolean; reason: string }> {
  const { judgeRecommendation } = await import('./llm-judge');
  const recLine = text.match(/^[*_]*\s*recommendation\s*[*_]*\s*:\s*(.+)$/im);
  const hadLiteralBecause = !!recLine && /\bbecause\s+\S/i.test(recLine[1]);

  let graded = text;
  if (recLine && !hadLiteralBecause) {
    // Rewrite "Recommendation: <choice><sep><reason>" → "...<choice> because <reason>"
    // sep ∈ {". ", " — ", " - ", ": "} right after a short choice token.
    const normalizedLine = recLine[1].replace(
      /^([^.:—-]{1,40}?)\s*(?:\.\s+|\s*[—-]\s+|:\s+)(\S.+)$/,
      '$1 because $2',
    );
    if (normalizedLine !== recLine[1]) {
      graded = text.replace(recLine[0], `Recommendation: ${normalizedLine}`);
    }
  }

  try {
    const r = await judgeRecommendation(graded);
    return { substance: r.reason_substance, present: r.present, hadLiteralBecause, reason: r.reason_text };
  } catch {
    return { substance: 0, present: !!recLine, hadLiteralBecause, reason: '' };
  }
}

/**
 * Build a throwaway plan dir holding a SPECIFIC plan-ceo-review SKILL.md (so we
 * can pit the carved skeleton against the verbose monolith). `sectionsFrom`, if
 * given, copies that dir's sections/ alongside (for the carved variant).
 */
export function setupPlanCeoDir(opts: {
  skillMd: string;
  sectionsFrom?: string | null;
  tmpPrefix?: string;
}): string {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), opts.tmpPrefix ?? 'auq-sdk-'));
  const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 });
  run('git', ['init', '-b', 'main']);
  run('git', ['config', 'user.email', 'test@test.com']);
  run('git', ['config', 'user.name', 'Test']);
  fs.writeFileSync(
    path.join(dir, 'plan.md'),
    [
      '# Plan: Launch a "developer-friendly" pricing tier',
      '',
      '## Goal',
      'Increase developer adoption.',
      '',
      '## Success metric',
      'More signups.',
      '',
      '## Premise',
      "We haven't talked to any developers about whether the current pricing is a",
      'barrier. The team agreed it "feels like" it should be cheaper.',
    ].join('\n'),
  );
  fs.mkdirSync(path.join(dir, 'plan-ceo-review'), { recursive: true });
  fs.writeFileSync(path.join(dir, 'plan-ceo-review', 'SKILL.md'), opts.skillMd);
  if (opts.sectionsFrom && fs.existsSync(opts.sectionsFrom)) {
    fs.cpSync(opts.sectionsFrom, path.join(dir, 'plan-ceo-review', 'sections'), { recursive: true });
  }
  run('git', ['add', '.']);
  run('git', ['commit', '-m', 'plan']);
  return dir;
}

/**
 * Generic: build a throwaway dir holding ANY skill's SKILL.md (+ optional
 * sections) plus arbitrary fixture files, so the matrix can drive each skill to
 * its first AUQ. Mirrors setupPlanCeoDir but skill-agnostic.
 */
export function setupSkillDir(opts: {
  skillName: string;
  skillMd: string;
  sectionsFrom?: string | null;
  fixtures?: Record<string, string>;
  tmpPrefix?: string;
}): string {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), opts.tmpPrefix ?? `auq-${opts.skillName}-`));
  const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 });
  run('git', ['init', '-b', 'main']);
  run('git', ['config', 'user.email', 'test@test.com']);
  run('git', ['config', 'user.name', 'Test']);
  for (const [name, content] of Object.entries(opts.fixtures ?? {})) {
    const p = path.join(dir, name);
    fs.mkdirSync(path.dirname(p), { recursive: true });
    fs.writeFileSync(p, content);
  }
  fs.mkdirSync(path.join(dir, opts.skillName), { recursive: true });
  fs.writeFileSync(path.join(dir, opts.skillName, 'SKILL.md'), opts.skillMd);
  if (opts.sectionsFrom && fs.existsSync(opts.sectionsFrom)) {
    fs.cpSync(opts.sectionsFrom, path.join(dir, opts.skillName, 'sections'), { recursive: true });
  }
  run('git', ['add', '.']);
  run('git', ['commit', '-m', 'fixture']);
  return dir;
}

/** Read any skill's current (worktree) SKILL.md + its sections dir if present. */
export function skillFromWorktree(skillName: string): { skillMd: string; sectionsFrom: string | null } {
  const sec = path.join(ROOT, skillName, 'sections');
  return {
    skillMd: fs.readFileSync(path.join(ROOT, skillName, 'SKILL.md'), 'utf-8'),
    sectionsFrom: fs.existsSync(sec) ? sec : null,
  };
}

/**
 * Generic: drive ANY skill to its FIRST AskUserQuestion and capture the
 * verbatim decision-brief text the model would have shown. `scenario` is the
 * per-skill prose that triggers a real AUQ (e.g. "review plan.md", "audit
 * vuln.ts for security"). Absolute skill path + Read/Write-only so the agent
 * cannot wander to the global install.
 */
export async function captureFirstAuq(opts: {
  planDir: string;
  skillName: string;
  scenario: string;
  testName: string;
  runId?: string;
  model?: string;
}): Promise<string> {
  const outFile = path.join(opts.planDir, 'ask-capture.md');
  const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
  const prompt = `You are running a format-capture test. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT search for, Glob, find, or read any other SKILL.md anywhere — especially nothing under ~/.claude or /Users.

Read ${skillPath} and follow its workflow for this scenario:

${opts.scenario}

This is a capture test, not an interactive session. Skip any system-audit / environment-setup / codebase-exploration steps. When you reach the FIRST point where the skill would call AskUserQuestion, write the verbatim full decision-brief text of that question (title, ELI10, stakes, recommendation, every option with its ✅/❌ pros/cons bullets, and the Net line) to ${outFile}. Do NOT call any tool to ask the user. Do NOT paraphrase. After writing the file, STOP.`;

  await runSkillTest({
    prompt,
    workingDirectory: opts.planDir,
    allowedTools: ['Read', 'Write'],
    maxTurns: 14,
    timeout: 240_000,
    testName: opts.testName,
    runId: opts.runId,
    model: resolveEvalModel('capture', opts.model),
  });

  try {
    return fs.readFileSync(outFile, 'utf-8');
  } catch {
    return '';
  }
}

/**
 * Drive ANY carved skill through a real `claude -p` run and detect, LOSSLESSLY,
 * which `sections/<file>.md` files the agent actually Read — from the tool-use
 * stream, not the ANSI screen buffer. This is the reliable replacement for the
 * real-PTY `visibleSince()` screen-scraping the section-loading tests used to do
 * (which silently saw nothing in a Conductor PTY: cursor-positioned renders and
 * an unanswered Step 0 question loop both defeat the regex).
 *
 * The skill under test is the planted copy in `planDir` (pin the absolute path so
 * the agent cannot wander to the global install). AskUserQuestion is declared
 * unavailable so the agent auto-picks the recommended option and proceeds far
 * enough to hit the post-Step-0 STOP-Read directives; Read is the tool a STOP-Read
 * resolves to, so Read/Grep/Glob/Write is all the agent needs (no Bash → it cannot
 * `find /` its way out, nor run git/gh mutations).
 */
export function hasDisabledOutsideReview(output: string): boolean {
  const headings = [...output.matchAll(/^## GSTACK REVIEW REPORT[ \t]*\r?$/gm)];
  const heading = headings.at(-1);
  if (!heading) return false;
  const section = output.slice(heading.index! + heading[0].length).split(/^##[ \t]+/m, 1)[0];
  const plain = (cell: string) => cell.replace(/[*_`]/g, '').trim().replace(/\s+/g, ' ').toLowerCase();
  for (const line of section.split('\n')) {
    if (!line.trimStart().startsWith('|')) continue;
    const cells = line.split('|').map(plain);
    if (cells[1] === 'outside review') {
      return /^disabled(?:$|\s|[(:—–-])/.test(cells[5] ?? '');
    }
  }
  return false;
}

export async function captureSectionReads(opts: {
  planDir: string;
  skillName: string;
  scenario: string;
  /** Relative filename the agent writes its final output to (terminal signal). */
  reportFile?: string;
  /** Marker proving a real report/plan was produced (default: any non-empty text). */
  reportMarker?: RegExp;
  testName: string;
  runId?: string;
  model?: string;
  maxTurns?: number;
  timeout?: number;
  /** Measure native section loading with the documented extra-review opt-out. */
  nativeReviewOnly?: boolean;
}): Promise<{ readSections: Set<string>; reportProduced: boolean; toolCalls: SkillTestResult['toolCalls']; output: string }> {
  const outFile = path.join(opts.planDir, opts.reportFile ?? 'REPORT.md');
  const skillPath = path.join(opts.planDir, opts.skillName, 'SKILL.md');
  // Outside-review dispatch has separate behavioral coverage. Native-only
  // captures use the real supported control in state owned by this call;
  // never mutate the operator's or another capture's gstack configuration.
  // Keep the model-facing config path relative to the fixture's working directory.
  const stateDir = opts.nativeReviewOnly
    ? fs.mkdtempSync(path.join(path.resolve(opts.planDir), '.gstack-section-state-')) : null;
  const nativeReviewRule = stateDir
    ? `\n- Read ${path.relative(path.resolve(opts.planDir), path.join(stateDir, 'config.yaml'))}, the isolated gstack configuration for this capture. It sets codex_reviews: disabled. Follow that documented control: skip the entire extra outside-review step, including its native fallback, and report outside coverage as disabled. Complete all native review sections and the full required report.`
    : '';
  // Preserve full method execution while avoiding a second written walkthrough
  // of decisions already represented in the amended plan and required outputs.
  const planReviewWritingRule = opts.skillName === 'plan-ceo-review'
    ? `\n- Write a concise, complete decision record: preserve original requirements and accepted plan amendments. Record each finding once with concrete evidence, the selected remedy, residual risks, and verification. Give all 11 sections an explicit outcome (including no issues or justified skips); retain the complete required registries, applicable diagrams, tasks, completion summary, and exact GSTACK REVIEW REPORT table. Cross-reference those records instead of repeating findings, option deliberations, diagrams, or registries in each section. Use compact outcome entries and short table cells; execute the review checklists without copying their questions or narrating every check into the artifact. Brevity must preserve every finding, accepted requirement, required field, and required diagram in its specified format. Do not expand the artifact into full implementation or test code unless that code is needed to specify an accepted plan change. This is a writing rule only: execute the full review, perform every required lazy-file Read, and complete all required artifacts before returning.`
    : '';
  const prompt = `You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users.

Read ${skillPath} and EXECUTE its workflow for this scenario:

${opts.scenario}

Rules for this run:
- Skip system-audit, environment-setup, telemetry, and codebase-exploration steps.
- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask.
- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/<file>\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory.
- Do NOT run git, gh, commit, push, or any mutating command.
- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${outFile}.${nativeReviewRule}${planReviewWritingRule}
- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`;

  let result: SkillTestResult;
  try {
    if (stateDir) fs.writeFileSync(path.join(stateDir, 'config.yaml'), 'codex_reviews: disabled\n');
    result = await runSkillTest({
      prompt,
      workingDirectory: opts.planDir,
      allowedTools: ['Read', 'Grep', 'Glob', 'Write'],
      tools: ['Read', 'Grep', 'Glob', 'Write'],
      publicStreamDiagnostics: true,
      maxTurns: opts.maxTurns ?? 25,
      timeout: opts.timeout ?? 300_000,
      testName: opts.testName,
      runId: opts.runId,
      model: resolveEvalModel('capture', opts.model),
      ...(stateDir ? { env: { GSTACK_HOME: stateDir, GSTACK_STATE_ROOT: stateDir } } : {}),
    });
  } finally {
    if (stateDir) fs.rmSync(stateDir, { recursive: true, force: true });
  }

  const readSections = new Set<string>();
  for (const c of result.toolCalls) {
    if (c.tool !== 'Read') continue;
    const fp = String(c.input?.file_path ?? '');
    const m = fp.match(/sections\/([A-Za-z0-9._-]+\.md)/);
    if (m) readSections.add(m[1]);
  }

  let output = '';
  try { output = fs.readFileSync(outFile, 'utf-8'); } catch { output = result.output ?? ''; }
  const reportProduced = result.exitReason === 'success'
    && (opts.reportMarker ? opts.reportMarker.test(output) : output.trim().length > 0);

  return { readSections, reportProduced, toolCalls: result.toolCalls, output };
}

/** Read the carved (current worktree) plan-ceo SKILL.md + its sections dir. */
export function carvedSkill(): { skillMd: string; sectionsFrom: string | null } {
  const sec = path.join(ROOT, 'plan-ceo-review', 'sections');
  return {
    skillMd: fs.readFileSync(path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), 'utf-8'),
    sectionsFrom: fs.existsSync(sec) ? sec : null,
  };
}

/** Read the pre-carve verbose monolith plan-ceo SKILL.md.
 *  VENDORED fixture (v1.75 precedent), not a git ref: the old default
 *  `git show ab66193e^:...` pinned a BRANCH-LOCAL commit — it dies the day
 *  that branch is pruned and already fails on shallow clones. The fixture
 *  is the frozen pre-cut render; test/git-ref-fixture-tripwire.test.ts
 *  keeps this class from coming back. */
export function verboseSkill(): string {
  return fs.readFileSync(
    path.join(ROOT, 'test', 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md'),
    'utf-8',
  );
}

function execGit(args: string[]): string {
  const r = spawnSync('git', args, { cwd: ROOT, encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024, timeout: 30_000 });
  if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${r.stderr}`);
  return r.stdout;
}

/**
 * Drive plan-ceo-review to its Step 0F mode-selection AskUserQuestion in the
 * given plan dir and capture the verbatim question text the model generates.
 * Returns the captured text ('' if the agent never wrote the file).
 */
export async function captureModeSelectionAuq(opts: {
  planDir: string;
  testName: string;
  runId?: string;
  model?: string;
}): Promise<string> {
  const outFile = path.join(opts.planDir, 'ask-capture.md');
  const skillPath = path.join(opts.planDir, 'plan-ceo-review', 'SKILL.md');
  const planPath = path.join(opts.planDir, 'plan.md');
  // CRITICAL: pin the EXACT skill file. Without this the agent runs
  // `find / -name SKILL.md` / Glob and reads the GLOBAL install
  // (~/.claude/skills/...) instead of the version-under-test in the temp dir —
  // which silently invalidates a carved-vs-verbose A/B (both sides end up
  // reading the same global skill). Absolute path + no-wander instruction +
  // Bash disallowed (so `find /` is impossible) locks it to the planted file.
  const prompt = `You are running a format-capture test. Use ONLY these two files:
  - The skill to follow: ${skillPath}
  - The plan to review: ${planPath}

Read ${skillPath} for the review workflow. Do NOT search for, Glob, find, or read any OTHER SKILL.md anywhere on the system — especially nothing under ~/.claude or /Users. The ONLY skill file you may read is the absolute path above.

Read ${planPath} — that is the plan to review. It is a standalone plan document, not a codebase. Skip any codebase exploration or system-audit steps.

Proceed to Step 0F (Mode Selection), where the skill presents the 4 review-mode options to the user via AskUserQuestion.

Write the verbatim text of that AskUserQuestion (the full decision brief: title, ELI10, stakes, recommendation, every option with its pros/cons bullets, and the Net line) to ${outFile}. Do NOT call any tool to ask the user. Do NOT paraphrase. After writing the file, stop.`;

  await runSkillTest({
    prompt,
    workingDirectory: opts.planDir,
    // Read + Write only: no Bash means the agent cannot `find /` its way to the
    // global install, and the skill's preamble bash blocks (irrelevant to format
    // capture) can't run and wander.
    allowedTools: ['Read', 'Write'],
    maxTurns: 12,
    timeout: 240_000,
    testName: opts.testName,
    runId: opts.runId,
    model: resolveEvalModel('capture', opts.model),
  });

  try {
    const text = fs.readFileSync(outFile, 'utf-8');
    // Defense in depth: verify the agent actually read the planted skill, not a
    // global one. If the captured run somehow read elsewhere we can't detect it
    // from the output file alone, so callers should also confirm via the run
    // log; this guard at least catches an empty/placeholder capture.
    return text;
  } catch {
    return '';
  }
}