gstack/test/helpers/providers/claude.ts

import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { resolveClaudeCommand } from '../../../lib/claude-bin';
import { resolveEvalModel } from '../../../lib/eval-model';

/**
 * Claude adapter — wraps the `claude` CLI via claude -p.
 *
 * For brevity and to avoid duplicating the full stream-json parser, this adapter
 * uses claude CLI in non-interactive mode (--print) with the simpler JSON output
 * format. If richer event-level metrics are needed (per-tool timing etc.),
 * swap to session-runner's full stream-json parser.
 */
export class ClaudeAdapter implements ProviderAdapter {
  readonly name = 'claude';
  readonly family = 'claude' as const;

  async available(): Promise<AvailabilityCheck> {
    // Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
    // resolver so Windows + override paths behave the same as production sites.
    const resolved = resolveClaudeCommand();
    if (!resolved) {
      return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
    }
    // Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY OR (macOS)
    // the Keychain entry subscription installs use instead of the creds file.
    // #1890: the default macOS install stores OAuth under the generic-password
    // service "Claude Code-credentials" and never writes .credentials.json,
    // so the file-or-env sniff reported "No Claude auth found" while
    // `claude -p` worked fine. Metadata probe only (no -w — never reads the
    // secret), and any failure of `security` itself falls through to the
    // not-found reason rather than throwing.
    const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
    const hasCreds = fs.existsSync(credsPath);
    const hasKey = !!process.env.ANTHROPIC_API_KEY;
    let hasKeychain = false;
    if (!hasCreds && !hasKey && process.platform === 'darwin') {
      try {
        const probe = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials'], {
          stdio: 'ignore',
          timeout: 5000,
        });
        hasKeychain = probe.status === 0;
      } catch {
        hasKeychain = false;
      }
    }
    if (!hasCreds && !hasKey && !hasKeychain) {
      return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
    }
    return { ok: true };
  }

  async run(opts: RunOpts): Promise<RunResult> {
    const start = Date.now();
    const resolved = resolveClaudeCommand();
    if (!resolved) {
      throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
    }
    const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
    const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
    args.push('--model', model);
    if (opts.extraArgs) args.push(...opts.extraArgs);

    try {
      const out = execFileSync(resolved.command, args, {
        input: opts.prompt,
        cwd: opts.workdir,
        timeout: opts.timeoutMs,
        encoding: 'utf-8',
        maxBuffer: 32 * 1024 * 1024,
        // Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
        // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
        env: { ...process.env, GSTACK_HEADLESS: '1' },
      });
      const parsed = this.parseOutput(out);
      return {
        output: parsed.output,
        tokens: parsed.tokens,
        durationMs: Date.now() - start,
        toolCalls: parsed.toolCalls,
        modelUsed: parsed.modelUsed || model,
      };
    } catch (err: unknown) {
      const durationMs = Date.now() - start;
      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
      const stderr = e.stderr?.toString() ?? '';
      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
      }
      if (/unauthorized|auth|login/i.test(stderr)) {
        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
      }
      if (/rate[- ]?limit|429/i.test(stderr)) {
        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
      }
      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
    }
  }

  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
    return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
  }

  /**
   * Parse claude -p --output-format json output. Shape (as of 2026-04):
   *   { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
   *     num_turns, session_id, ... }
   * Older formats may differ — adapter is best-effort.
   */
  private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
    try {
      const obj = JSON.parse(raw);
      const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
      const u = obj.usage ?? {};
      return {
        output: result,
        tokens: {
          input: u.input_tokens ?? 0,
          output: u.output_tokens ?? 0,
          cached: u.cache_read_input_tokens,
        },
        toolCalls: obj.num_turns ?? 0,
        modelUsed: obj.model,
      };
    } catch {
      // Non-JSON output: treat as plain text.
      return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
    }
  }

  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
    return {
      output: '',
      tokens: { input: 0, output: 0 },
      durationMs,
      toolCalls: 0,
      modelUsed: model ?? resolveEvalModel('capture'),
      error,
    };
  }
}