gstack/test/helpers/providers/claude.ts

import {
  csoProducerChildEnvironment,
  csoProducerHelperHome,
  csoProducerHelperLauncher,
  csoProducerProviderCommand,
  csoProducerStateDirectory,
  type ProviderAdapter,
  type RunOpts,
  type RunResult,
  type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { resolveClaudeCommand } from '../../../lib/claude-bin';
import { resolveEvalModel } from '../../../lib/eval-model';
import { runClaudeProducerStream } from './claude-stream';

/**
 * Claude adapter — wraps the `claude` CLI via claude -p.
 *
 * For brevity and to avoid duplicating the full stream-json parser, this adapter
 * uses claude CLI in non-interactive mode (--print) with the simpler JSON output
 * format. CSO producers stream events instead (claude-stream.ts) so a timed-out
 * run still reports its usage.
 */
export class ClaudeAdapter implements ProviderAdapter {
  readonly name = 'claude';
  readonly family = 'claude' as const;

  async available(opts?: RunOpts): Promise<AvailabilityCheck> {
    // Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
    // resolver so Windows + override paths behave the same as production sites.
    const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
    const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
    if (!resolved) {
      return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
    }
    // Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY OR (macOS)
    // the Keychain entry subscription installs use instead of the creds file.
    // #1890: the default macOS install stores OAuth under the generic-password
    // service "Claude Code-credentials" and never writes .credentials.json,
    // so the file-or-env sniff reported "No Claude auth found" while
    // `claude -p` worked fine. Metadata probe only (no -w — never reads the
    // secret), and any failure of `security` itself falls through to the
    // not-found reason rather than throwing.
    const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
    const hasCreds = !opts?.csoProducer && fs.existsSync(credsPath);
    const hasKey = !!(process.env.ANTHROPIC_API_KEY || process.env.CLAUDE_CODE_OAUTH_TOKEN);
    let hasKeychain = false;
    if (!hasCreds && !hasKey && process.platform === 'darwin') {
      try {
        const probe = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials'], {
          stdio: 'ignore',
          timeout: 5000,
        });
        hasKeychain = probe.status === 0;
      } catch {
        hasKeychain = false;
      }
    }
    if (!hasCreds && !hasKey && !hasKeychain) {
      return { ok: false, reason: opts?.csoProducer
        ? 'No Claude producer auth found. Export ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN; macOS Keychain auth is also supported.'
        : 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
    }
    return { ok: true };
  }

  async run(opts: RunOpts): Promise<RunResult> {
    const start = Date.now();
    const producerCommand = csoProducerProviderCommand(opts);
    const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
    if (!resolved) {
      throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
    }
    const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
    const stateDirectory = csoProducerStateDirectory(opts);
    const args = claudeExecArgs(opts, model, resolved.argsPrefix);
    const env = claudeExecEnvironment(opts, process.env);
    if (stateDirectory) {
      assertClaudeProducerPrerequisites();
      assertClaudeProducerSandboxPolicy(args, env, csoProducerHelperLauncher(opts)!);
    }

    try {
      if (stateDirectory) {
        prepareClaudeProducerState(stateDirectory);
        return await runClaudeProducerStream(resolved.command, args, { input: opts.prompt, cwd: claudeExecWorkingDirectory(opts), env, timeoutMs: opts.timeoutMs, model });
      }
      const out = execFileSync(resolved.command, args, {
        input: opts.prompt,
        cwd: claudeExecWorkingDirectory(opts),
        timeout: opts.timeoutMs,
        encoding: 'utf-8',
        maxBuffer: 32 * 1024 * 1024,
        // Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
        // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
        env,
      });
      return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
    } catch (err: unknown) {
      const durationMs = Date.now() - start;
      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
      const stderr = e.stderr?.toString() ?? '';
      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
      }
      if (/unauthorized|auth|login/i.test(stderr)) {
        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
      }
      if (/rate[- ]?limit|429/i.test(stderr)) {
        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
      }
      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
    } finally {
      if (stateDirectory) removeClaudeProducerState(stateDirectory);
    }
  }

  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
    return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
  }

  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
    return {
      output: '',
      tokens: { input: 0, output: 0 },
      durationMs,
      toolCalls: 0,
      modelUsed: model ?? resolveEvalModel('capture'),
      error,
    };
  }
}

/** Map Claude JSON output; paid producers require the documented nonblank `result` envelope. */
export function resultFromClaudeOutput(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
  const durationMs=opts.durationMs??0,defaultModel=opts.model??resolveEvalModel('capture');
  try{
    const obj=JSON.parse(raw),explicitError=obj?.is_error===true||(typeof obj?.subtype==='string'&&obj.subtype!=='success'),validResult=!explicitError&&typeof obj?.result==='string'&&obj.result.trim().length>0;
    if(opts.producer&&!validResult)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
    const output=typeof obj?.result==='string'?obj.result:String(obj?.result??''),usage=obj?.usage??{};
    return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel};
  }catch{
    if(opts.producer)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
    return{output:raw,tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel};
  }
}

export interface ClaudeProducerPaths { root: string; home: string; workdir: string }

export function assertClaudeProducerPrerequisites(): void {
  if (process.platform !== 'linux') return;
  const executable = '/usr/bin/bwrap';
  let stat: fs.Stats;
  try { stat = fs.lstatSync(executable); } catch { throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP'); }
  if (!stat.isFile() || stat.isSymbolicLink() || fs.realpathSync(executable) !== executable || stat.uid !== 0 || (stat.mode & 0o022) !== 0 || (stat.mode & 0o111) === 0) {
    throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP');
  }
}

export function claudeProducerPaths(stateDirectory: string): ClaudeProducerPaths {
  const root = path.join(stateDirectory, 'claude-provider');
  return { root, home: path.join(root, 'home'), workdir: path.join(root, 'work') };
}

export function prepareClaudeProducerState(stateDirectory: string): ClaudeProducerPaths {
  const paths = claudeProducerPaths(stateDirectory);
  if (fs.existsSync(paths.root)) throw new Error('CLAUDE_PRODUCER_STATE_EXISTS');
  fs.mkdirSync(paths.root, { mode: 0o700 });
  fs.mkdirSync(paths.home, { mode: 0o700 });
  fs.mkdirSync(paths.workdir, { mode: 0o700 });
  return paths;
}

export function removeClaudeProducerState(stateDirectory: string): void {
  fs.rmSync(claudeProducerPaths(stateDirectory).root, { recursive: true, force: true });
}

export function claudeExecWorkingDirectory(opts: RunOpts): string {
  const stateDirectory = csoProducerStateDirectory(opts);
  return stateDirectory ? claudeProducerPaths(stateDirectory).workdir : opts.workdir;
}

export function claudeProducerTools(opts: RunOpts): string {
  const launcher = csoProducerHelperLauncher(opts);
  if (!launcher) throw new Error('CSO producer helper launcher is required');
  return [`Bash(${launcher})`, `Bash(${launcher} *)`, 'Write'].join(',');
}

/** Built-in tools: users' /cso starts its independent reviewers with Agent; subagents inherit this whole policy. */
export const CLAUDE_PRODUCER_BUILTIN_TOOLS = 'Bash,Write,Agent';

/** Claude's sandbox wraps every Bash command except the trusted launcher, which must run where users run it. */
export function claudeProducerSettings(launcher: string): { sandbox: Record<string, unknown> } {
  return {
    sandbox: {
      enabled: true,
      failIfUnavailable: true,
      allowUnsandboxedCommands: false,
      autoAllowBashIfSandboxed: false,
      excludedCommands: [launcher, `${launcher} *`],
    },
  };
}

export function assertClaudeProducerSandboxPolicy(args: readonly string[], env: Record<string, string>, launcher: string): void {
  const fail = (detail: string): never => { throw new Error(`CLAUDE_PRODUCER_SANDBOX_POLICY_MISMATCH: ${detail}`); };
  const value = (flag: string): string => {
    const positions = args.flatMap((arg, index) => arg === flag || arg.startsWith(`${flag}=`) ? [index] : []);
    if (positions.length !== 1 || args[positions[0]] !== flag || positions[0] + 1 >= args.length) fail(`expected exactly one ${flag} <value>`);
    return args[positions[0] + 1];
  };
  if (value('--tools') !== CLAUDE_PRODUCER_BUILTIN_TOOLS) fail(`--tools must be exactly ${CLAUDE_PRODUCER_BUILTIN_TOOLS}`);
  if (value('--allowed-tools') !== `Bash(${launcher}),Bash(${launcher} *),Write`) fail('--allowed-tools must name only the launcher and Write');
  if (value('--permission-mode') !== 'dontAsk') fail('--permission-mode must be dontAsk');
  for (const flag of ['--restricted', '--safe-mode', '--strict-mcp-config']) if (args.filter(arg => arg === flag).length !== 1) fail(`expected exactly one ${flag}`);
  const widening = args.find(arg => /^--(mcp-config|agents?|plugin-dir|dangerously-skip-permissions|allow-dangerously-skip-permissions|disallowed-tools|disallowedTools|setting-sources)(=|$)/.test(arg));
  if (widening) fail(`${widening.split('=')[0]} would change what the agent and its subagents may do`);
  const raw = value('--settings');
  let settings: unknown;
  try { settings = JSON.parse(raw); } catch { fail('--settings is not JSON'); }
  const expected = { enabled: true, failIfUnavailable: true, allowUnsandboxedCommands: false, autoAllowBashIfSandboxed: false, excludedCommands: [launcher, `${launcher} *`] };
  const sandbox = (settings as { sandbox?: Record<string, unknown> } | null)?.sandbox;
  if (!settings || typeof settings !== 'object' || Object.keys(settings).join() !== 'sandbox' || !sandbox || typeof sandbox !== 'object') fail('--settings must hold only a sandbox object');
  if (Object.keys(sandbox!).sort().join() !== Object.keys(expected).sort().join()) fail('sandbox settings keys differ from the launcher-only exclusion policy');
  for (const [key, wanted] of Object.entries(expected)) {
    if (JSON.stringify(sandbox![key]) !== JSON.stringify(wanted)) fail(`sandbox.${key} must be ${JSON.stringify(wanted)}`);
  }
  if (env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB !== undefined) fail('CLAUDE_CODE_SUBPROCESS_ENV_SCRUB forces every command, the launcher included, into the sandbox');
}

export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonly string[] = []): string[] {
  const stateDirectory = csoProducerStateDirectory(opts);
  // Producers stream events so a timed-out or killed run still reports the usage it spent.
  const output = stateDirectory ? ['stream-json', '--verbose', '--include-partial-messages'] : ['json'];
  const args = [...argsPrefix, '-p', '--output-format', ...output, '--model', model];
  if (stateDirectory) {
    if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
    const tools = claudeProducerTools(opts);
    const helperHome = csoProducerHelperHome(opts)!;
    args.push(
      '--restricted',
      '--safe-mode',
      '--no-session-persistence',
      '--permission-prompts', 'none',
      '--permission-mode', 'dontAsk',
      '--tools', CLAUDE_PRODUCER_BUILTIN_TOOLS,
      '--allowed-tools', tools,
      '--add-dir', claudeProducerPaths(stateDirectory).workdir,
      '--add-dir', helperHome,
      '--settings', JSON.stringify(claudeProducerSettings(csoProducerHelperLauncher(opts)!)),
      '--strict-mcp-config',
      '--no-chrome',
    );
  }
  if (opts.extraArgs) args.push(...opts.extraArgs);
  return args;
}

export function claudeExecEnvironment(
  opts: RunOpts,
  source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
  const stateDirectory = csoProducerStateDirectory(opts);
  const paths = stateDirectory ? claudeProducerPaths(stateDirectory) : undefined;
  return {
    ...(opts.csoProducer ? csoProducerChildEnvironment('claude', source) : source),
    ...(paths ? { HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! } : {}),
    GSTACK_HEADLESS: '1',
  } as Record<string, string>;
}