refactor: capture plain-text CLI output, drop hardcoded provider schemas

The adapters parsed each vendor's proprietary JSON stream (Claude json,
Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a
per-model pricing table turned tokens into cost. That coupling was the
brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor
reshuffled its output, and it duplicated what any tool that instruments the
real model call already does. Braintrust owns scoring; it can't see a CLI
subprocess's tokens anyway, so computing cost ourselves meant maintaining
both a parser and a price table forever.

Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring
is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls;
the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all
three JSON parsers, and the Gemini stream-schema parser + its test. Gemini
auth detection (env/OAuth/.env) is kept — that's not schema parsing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/model-benchmark/providers/claude.ts

@@ -1,5 +1,4 @@
-import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
-import { estimateCostUsd } from '../pricing';
+import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
 import { execFileSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -7,12 +6,10 @@ import * as os from 'os';
 import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
 
 /**
- * Claude adapter — wraps the `claude` CLI via claude -p.
+ * Claude adapter — wraps the `claude` CLI via `claude -p` in plain-text mode.
  *
- * For brevity and to avoid duplicating the full stream-json parser, this adapter
- * uses claude CLI in non-interactive mode (--print) with the simpler JSON output
- * format. If richer event-level metrics are needed (per-tool timing etc.),
- * swap to session-runner's full stream-json parser.
+ * We capture stdout as the answer and do not parse Claude's JSON output shape:
+ * scoring is Braintrust's job and it only needs the text.
  */
 export class ClaudeAdapter implements ProviderAdapter {
   readonly name = 'claude';
@@ -41,7 +38,7 @@ export class ClaudeAdapter implements ProviderAdapter {
     if (!resolved) {
       throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
     }
-    const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
+    const args = [...resolved.argsPrefix, '-p'];
     if (opts.model) args.push('--model', opts.model);
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
@@ -56,70 +53,25 @@ export class ClaudeAdapter implements ProviderAdapter {
         // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
         env: { ...process.env, GSTACK_HEADLESS: '1' },
       });
-      const parsed = this.parseOutput(out);
       return {
-        output: parsed.output,
-        tokens: parsed.tokens,
+        output: out.trim(),
         durationMs: Date.now() - start,
-        toolCalls: parsed.toolCalls,
-        modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
+        modelUsed: opts.model ?? 'claude',
       };
     } catch (err: unknown) {
-      const durationMs = Date.now() - start;
-      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
-      const stderr = e.stderr?.toString() ?? '';
-      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
-        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
-      }
-      if (/unauthorized|auth|login/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      if (/rate[- ]?limit|429/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
+      return this.errorResult(Date.now() - start, err, opts.model);
     }
   }
 
-  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
-    return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
-  }
-
-  /**
-   * Parse claude -p --output-format json output. Shape (as of 2026-04):
-   *   { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
-   *     num_turns, session_id, ... }
-   * Older formats may differ — adapter is best-effort.
-   */
-  private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
-    try {
-      const obj = JSON.parse(raw);
-      const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
-      const u = obj.usage ?? {};
-      return {
-        output: result,
-        tokens: {
-          input: u.input_tokens ?? 0,
-          output: u.output_tokens ?? 0,
-          cached: u.cache_read_input_tokens,
-        },
-        toolCalls: obj.num_turns ?? 0,
-        modelUsed: obj.model,
-      };
-    } catch {
-      // Non-JSON output: treat as plain text.
-      return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
-    }
-  }
-
-  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
-    return {
-      output: '',
-      tokens: { input: 0, output: 0 },
-      durationMs,
-      toolCalls: 0,
-      modelUsed: model ?? 'claude-opus-4-7',
-      error,
-    };
+  private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
+    const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
+    const stderr = e.stderr?.toString() ?? '';
+    let code: RunError;
+    if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
+    else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
+    else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
+    else code = 'unknown';
+    const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
+    return { output: '', durationMs, modelUsed: model ?? 'claude', error: { code, reason } };
   }
 }

lib/model-benchmark/providers/gemini.ts

@@ -1,18 +1,15 @@
-import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
-import { estimateCostUsd } from '../pricing';
+import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
 import { execFileSync, spawnSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 
 /**
- * Gemini adapter — wraps the `gemini` CLI.
+ * Gemini adapter — wraps the `gemini` CLI in plain-text mode.
  *
- * Gemini CLI auth comes from its OAuth files, ~/.gemini/.env, or the
- * GOOGLE_API_KEY/GEMINI_API_KEY environment variables. Output
- * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
- * stream-json` is requested. This adapter uses a single-response form for simplicity
- * in benchmarks; richer streaming lives in gemini-session-runner.ts.
+ * Auth comes from its OAuth files, ~/.gemini/.env, or GOOGLE_API_KEY/GEMINI_API_KEY.
+ * We capture stdout as the answer and do not parse Gemini's stream-json schema —
+ * that coupling broke every time Gemini reshuffled its event shape.
  */
 export class GeminiAdapter implements ProviderAdapter {
   readonly name = 'gemini';
@@ -39,9 +36,10 @@ export class GeminiAdapter implements ProviderAdapter {
 
   async run(opts: RunOpts): Promise<RunResult> {
     const start = Date.now();
-    // Benchmarks are report-only. Plan mode keeps the CLI non-interactive while
-    // preventing a benchmark prompt from mutating the target worktree.
-    const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--approval-mode', 'plan', '--skip-trust'];
+    // --skip-trust lets the CLI run in disposable/non-git benchmark workdirs.
+    // Plain -p is non-interactive; benchmark prompts are reasoning tasks, not file
+    // ops, and the workdir is a throwaway temp dir.
+    const args = ['-p', opts.prompt, '--skip-trust'];
     if (opts.model) args.push('--model', opts.model);
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
@@ -52,83 +50,25 @@ export class GeminiAdapter implements ProviderAdapter {
         encoding: 'utf-8',
         maxBuffer: 32 * 1024 * 1024,
       });
-      const parsed = parseGeminiStreamJson(out);
       return {
-        output: parsed.output,
-        tokens: parsed.tokens,
+        output: out.trim(),
         durationMs: Date.now() - start,
-        toolCalls: parsed.toolCalls,
-        modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
+        modelUsed: opts.model ?? 'gemini',
       };
     } catch (err: unknown) {
-      const durationMs = Date.now() - start;
-      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
-      const stderr = e.stderr?.toString() ?? '';
-      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
-        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
-      }
-      if (/unauthorized|auth|login|api key/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      if (/rate[- ]?limit|429|quota/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
+      return this.errorResult(Date.now() - start, err, opts.model);
     }
   }
 
-  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
-    return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
-  }
-
-  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
-    return {
-      output: '',
-      tokens: { input: 0, output: 0 },
-      durationMs,
-      toolCalls: 0,
-      modelUsed: model ?? 'gemini-2.5-pro',
-      error,
-    };
+  private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
+    const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
+    const stderr = e.stderr?.toString() ?? '';
+    let code: RunError;
+    if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
+    else if (/unauthorized|auth|login|api key/i.test(stderr)) code = 'auth';
+    else if (/rate[- ]?limit|429|quota/i.test(stderr)) code = 'rate_limit';
+    else code = 'unknown';
+    const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
+    return { output: '', durationMs, modelUsed: model ?? 'gemini', error: { code, reason } };
   }
 }
-
-/** Parse both legacy `usage` events and current Gemini CLI `stats` events. */
-export function parseGeminiStreamJson(raw: string): {
-  output: string;
-  tokens: { input: number; output: number };
-  toolCalls: number;
-  modelUsed?: string;
-} {
-    let output = '';
-    let input = 0;
-    let out = 0;
-    let toolCalls = 0;
-    let modelUsed: string | undefined;
-    for (const line of raw.split('\n')) {
-      const s = line.trim();
-      if (!s) continue;
-      try {
-        const obj = JSON.parse(s);
-        if (obj.type === 'init' && typeof obj.model === 'string' && obj.model !== 'auto') {
-          modelUsed = obj.model;
-        } else if (obj.type === 'message' && obj.role === 'assistant') {
-          const content = typeof obj.content === 'string' ? obj.content : obj.text;
-          if (typeof content === 'string') output += content;
-        } else if (obj.type === 'tool_use') {
-          toolCalls += 1;
-        } else if (obj.type === 'result') {
-          const u = obj.usage ?? obj.stats ?? {};
-          input += u.input_token_count ?? u.prompt_tokens ?? u.input_tokens ?? u.input ?? 0;
-          out += u.output_token_count ?? u.completion_tokens ?? u.output_tokens ?? 0;
-          if (obj.model) modelUsed = obj.model;
-          if (!modelUsed && u.models && typeof u.models === 'object') {
-            modelUsed = Object.keys(u.models)[0];
-          }
-        }
-      } catch {
-        // skip malformed lines
-      }
-    }
-  return { output, tokens: { input, output: out }, toolCalls, modelUsed };
-}

lib/model-benchmark/providers/gpt.ts

@@ -1,16 +1,13 @@
-import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
-import { estimateCostUsd } from '../pricing';
+import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
 import { execFileSync, spawnSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 
 /**
- * GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
+ * GPT adapter — wraps the OpenAI `codex` CLI (`codex exec`) in plain-text mode.
  *
- * Codex uses ~/.codex/ for auth (not OPENAI_API_KEY). The --json flag emits
- * JSONL events; we parse `turn.completed` for usage and `agent_message` / etc.
- * for output aggregation.
+ * Captures stdout as the answer; no JSON-event parsing. Scoring is Braintrust's job.
  */
 export class GptAdapter implements ProviderAdapter {
   readonly name = 'gpt';
@@ -36,7 +33,7 @@ export class GptAdapter implements ProviderAdapter {
     // often run in temp dirs / non-git paths), so the read-only sandbox is now
     // the only boundary preventing codex from mutating the workdir. If you ever
     // remove `-s read-only`, drop `--skip-git-repo-check` too.
-    const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
+    const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check'];
     if (opts.model) args.push('-m', opts.model);
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
@@ -47,81 +44,25 @@ export class GptAdapter implements ProviderAdapter {
         encoding: 'utf-8',
         maxBuffer: 32 * 1024 * 1024,
       });
-      const parsed = this.parseJsonl(out);
       return {
-        output: parsed.output,
-        tokens: parsed.tokens,
+        output: out.trim(),
         durationMs: Date.now() - start,
-        toolCalls: parsed.toolCalls,
-        modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
+        modelUsed: opts.model ?? 'gpt',
       };
     } catch (err: unknown) {
-      const durationMs = Date.now() - start;
-      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
-      const stderr = e.stderr?.toString() ?? '';
-      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
-        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
-      }
-      if (/unauthorized|auth|login/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      if (/rate[- ]?limit|429/i.test(stderr)) {
-        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
-      }
-      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
+      return this.errorResult(Date.now() - start, err, opts.model);
     }
   }
 
-  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
-    return estimateCostUsd(tokens, model ?? 'gpt-5.4');
-  }
-
-  /**
-   * Parse codex exec --json JSONL stream.
-   * Key events:
-   *   - item.completed with item.type === 'agent_message' → text output
-   *   - item.completed with item.type === 'command_execution' → tool call
-   *   - turn.completed → usage.input_tokens, usage.output_tokens
-   *   - thread.started → session id (not used here)
-   */
-  private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
-    let output = '';
-    let input = 0;
-    let out = 0;
-    let toolCalls = 0;
-    let modelUsed: string | undefined;
-    for (const line of raw.split('\n')) {
-      const s = line.trim();
-      if (!s) continue;
-      try {
-        const obj = JSON.parse(s);
-        if (obj.type === 'item.completed' && obj.item) {
-          if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
-            output += (output ? '\n' : '') + obj.item.text;
-          } else if (obj.item.type === 'command_execution') {
-            toolCalls += 1;
-          }
-        } else if (obj.type === 'turn.completed') {
-          const u = obj.usage ?? {};
-          input += u.input_tokens ?? 0;
-          out += u.output_tokens ?? 0;
-          if (obj.model) modelUsed = obj.model;
-        }
-      } catch {
-        // skip malformed lines — codex stderr can leak in
-      }
-    }
-    return { output, tokens: { input, output: out }, toolCalls, modelUsed };
-  }
-
-  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
-    return {
-      output: '',
-      tokens: { input: 0, output: 0 },
-      durationMs,
-      toolCalls: 0,
-      modelUsed: model ?? 'gpt-5.4',
-      error,
-    };
+  private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
+    const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
+    const stderr = e.stderr?.toString() ?? '';
+    let code: RunError;
+    if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
+    else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
+    else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
+    else code = 'unknown';
+    const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
+    return { output: '', durationMs, modelUsed: model ?? 'gpt', error: { code, reason } };
   }
 }

lib/model-benchmark/providers/types.ts

@@ -1,8 +1,12 @@
 /**
  * Provider adapter interface — uniform contract for Claude, GPT, Gemini.
  *
- * Each adapter normalizes its provider's result shape into the RunResult below.
- * The benchmark runner only talks to adapters through this interface.
+ * Adapters shell out to the provider's CLI and return its plain-text output. We
+ * deliberately do NOT parse each vendor's proprietary JSON stream for tokens or
+ * cost: that coupling is brittle (it breaks every time a vendor reshuffles its
+ * output) and redundant — Braintrust owns scoring, and token/cost tracking
+ * belongs to whatever instruments the actual model call. The CLI's text answer
+ * is all the benchmark needs.
  */
 
 export interface RunOpts {
@@ -18,13 +22,6 @@ export interface RunOpts {
   extraArgs?: string[];
 }
 
-export interface TokenUsage {
-  input: number;
-  output: number;
-  /** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */
-  cached?: number;
-}
-
 export type RunError =
   | 'auth'       // Credentials missing or invalid.
   | 'timeout'    // Exceeded timeoutMs.
@@ -33,17 +30,13 @@ export type RunError =
   | 'unknown';   // Catch-all with reason populated.
 
 export interface RunResult {
-  /** Provider's textual output for the prompt. */
+  /** Provider's plain-text output for the prompt. */
   output: string;
-  /** Normalized token usage. 0s if unreported. */
-  tokens: TokenUsage;
   /** Wall-clock duration. */
   durationMs: number;
-  /** Count of tool/function calls made during the run (0 if unsupported). */
-  toolCalls: number;
-  /** Actual model ID the provider reports using (may be a variant of the family). */
+  /** Model label — the requested model or the family name. Not parsed from output. */
   modelUsed: string;
-  /** If the run failed, error code + human reason. output/tokens may be partial. */
+  /** If the run failed, error code + human reason. output may be empty/partial. */
   error?: { code: RunError; reason: string };
 }
 
@@ -67,6 +60,4 @@ export interface ProviderAdapter {
   available(): Promise<AvailabilityCheck>;
   /** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
   run(opts: RunOpts): Promise<RunResult>;
-  /** Estimate USD cost for the reported token usage and model. */
-  estimateCost(tokens: TokenUsage, model?: string): number;
 }