gstack/test/helpers/session-runner.ts

/**
 * Claude CLI subprocess runner for skill E2E testing.
 *
 * Spawns `claude -p` as a completely independent process (not via Agent SDK),
 * so it works inside Claude Code sessions. Pipes prompt via stdin, collects
 * JSON output, scans for browse errors.
 */

import * as fs from 'fs';
import * as path from 'path';

export interface CostEstimate {
  inputChars: number;
  outputChars: number;
  estimatedTokens: number;
  estimatedCost: number;  // USD
  turnsUsed: number;
}

export interface SkillTestResult {
  messages: any[];
  toolCalls: Array<{ tool: string; input: any; output: string }>;
  browseErrors: string[];
  exitReason: string;
  duration: number;
  output: string;
  costEstimate: CostEstimate;
}

const BROWSE_ERROR_PATTERNS = [
  /Unknown command: \w+/,
  /Unknown snapshot flag: .+/,
  /Exit code 1/,
  /ERROR: browse binary not found/,
  /Server failed to start/,
];

export async function runSkillTest(options: {
  prompt: string;
  workingDirectory: string;
  maxTurns?: number;
  allowedTools?: string[];
  timeout?: number;
}): Promise<SkillTestResult> {
  const {
    prompt,
    workingDirectory,
    maxTurns = 15,
    allowedTools = ['Bash', 'Read', 'Write'],
    timeout = 120_000,
  } = options;

  const startTime = Date.now();

  // Spawn claude -p with JSON output. Prompt piped via stdin to avoid
  // shell escaping issues. Env is passed through (child claude strips
  // its own parent-detection vars internally).
  const args = [
    '-p',
    '--output-format', 'json',
    '--dangerously-skip-permissions',
    '--max-turns', String(maxTurns),
    '--allowed-tools', ...allowedTools,
  ];

  // Write prompt to a temp file and pipe it via shell to avoid stdin buffering issues
  const promptFile = path.join(workingDirectory, '.prompt-tmp');
  fs.writeFileSync(promptFile, prompt);

  const proc = Bun.spawn(['sh', '-c', `cat "${promptFile}" | claude ${args.map(a => `"${a}"`).join(' ')}`], {
    cwd: workingDirectory,
    stdout: 'pipe',
    stderr: 'pipe',
  });

  // Race against timeout
  let stdout = '';
  let stderr = '';
  let exitReason = 'unknown';
  let timedOut = false;

  const timeoutId = setTimeout(() => {
    timedOut = true;
    proc.kill();
  }, timeout);

  try {
    const [outBuf, errBuf] = await Promise.all([
      new Response(proc.stdout).text(),
      new Response(proc.stderr).text(),
    ]);
    stdout = outBuf;
    stderr = errBuf;

    const exitCode = await proc.exited;
    clearTimeout(timeoutId);

    if (timedOut) {
      exitReason = 'timeout';
    } else if (exitCode === 0) {
      exitReason = 'success';
    } else {
      exitReason = `exit_code_${exitCode}`;
    }
  } catch (err: any) {
    clearTimeout(timeoutId);
    exitReason = timedOut ? 'timeout' : `error: ${err.message}`;
  } finally {
    try { fs.unlinkSync(promptFile); } catch { /* non-fatal */ }
  }

  const duration = Date.now() - startTime;

  // Parse JSON output
  let messages: any[] = [];
  let toolCalls: SkillTestResult['toolCalls'] = [];
  const browseErrors: string[] = [];
  let result: any = null;

  try {
    // stdout may have stderr warnings prefixed (e.g., "[WARN] Fast mode...")
    // Find the JSON object in the output
    const jsonStart = stdout.indexOf('{');
    if (jsonStart >= 0) {
      result = JSON.parse(stdout.slice(jsonStart));
    }
  } catch { /* non-JSON output */ }

  // Scan all output for browse errors
  const allText = stdout + '\n' + stderr;
  for (const pattern of BROWSE_ERROR_PATTERNS) {
    const match = allText.match(pattern);
    if (match) {
      browseErrors.push(match[0].slice(0, 200));
    }
  }

  // If JSON parsed, use the structured result
  if (result) {
    // Check result type for success
    if (result.type === 'result' && result.subtype === 'success') {
      exitReason = 'success';
    } else if (result.type === 'result' && result.subtype) {
      exitReason = result.subtype;
    }
  }

  // Save transcript on failure
  if (browseErrors.length > 0 || exitReason !== 'success') {
    try {
      const transcriptDir = path.join(workingDirectory, '.gstack', 'test-transcripts');
      fs.mkdirSync(transcriptDir, { recursive: true });
      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
      fs.writeFileSync(
        path.join(transcriptDir, `e2e-${timestamp}.json`),
        JSON.stringify({
          prompt: prompt.slice(0, 500),
          exitReason,
          browseErrors,
          duration,
          stderr: stderr.slice(0, 2000),
          result: result ? { type: result.type, subtype: result.subtype, result: result.result?.slice?.(0, 500) } : null,
        }, null, 2),
      );
    } catch { /* non-fatal */ }
  }

  // Cost from JSON result (exact) or estimate from chars
  const turnsUsed = result?.num_turns || 0;
  const estimatedCost = result?.total_cost_usd || 0;
  const inputChars = prompt.length;
  const outputChars = (result?.result || stdout).length;
  const estimatedTokens = (result?.usage?.input_tokens || 0)
    + (result?.usage?.output_tokens || 0)
    + (result?.usage?.cache_read_input_tokens || 0);

  const costEstimate: CostEstimate = {
    inputChars,
    outputChars,
    estimatedTokens,
    estimatedCost: Math.round((estimatedCost) * 100) / 100,
    turnsUsed,
  };

  return { messages, toolCalls, browseErrors, exitReason, duration, output: result?.result || stdout, costEstimate };
}