refactor: capture plain-text CLI output, drop hardcoded provider schemas

The adapters parsed each vendor's proprietary JSON stream (Claude json,
Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a
per-model pricing table turned tokens into cost. That coupling was the
brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor
reshuffled its output, and it duplicated what any tool that instruments the
real model call already does. Braintrust owns scoring; it can't see a CLI
subprocess's tokens anyway, so computing cost ourselves meant maintaining
both a parser and a price table forever.

Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring
is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls;
the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all
three JSON parsers, and the Gemini stream-schema parser + its test. Gemini
auth detection (env/OAuth/.env) is kept — that's not schema parsing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

test/benchmark-production-boundary.test.ts

@@ -42,8 +42,6 @@ test('production modules do not import from test directories', () => {
 
 test('former test-helper paths re-export the production benchmark API', async () => {
   const [
-    pricing,
-    helperPricing,
     claude,
     helperClaude,
     gpt,
@@ -51,8 +49,6 @@ test('former test-helper paths re-export the production benchmark API', async ()
     gemini,
     helperGemini,
   ] = await Promise.all([
-    import('../lib/model-benchmark/pricing'),
-    import('./helpers/pricing'),
     import('../lib/model-benchmark/providers/claude'),
     import('./helpers/providers/claude'),
     import('../lib/model-benchmark/providers/gpt'),
@@ -61,7 +57,6 @@ test('former test-helper paths re-export the production benchmark API', async ()
     import('./helpers/providers/gemini'),
   ]);
 
-  expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd);
   expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
   expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
   expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);

test/benchmark-runner.test.ts

@@ -1,44 +1,14 @@
 /**
- * Unit tests for benchmark pricing + tool-compatibility helpers.
+ * Unit tests for benchmark tool-compatibility helpers.
  *
- * Orchestration, scoring, and reporting moved to Braintrust
- * (lib/model-benchmark/braintrust-eval.ts); those are covered by
- * model-benchmark-braintrust.test.ts and the live e2e suite.
+ * Orchestration and scoring moved to Braintrust (lib/model-benchmark/braintrust-eval.ts);
+ * per-provider token/cost tracking was removed with the JSON-schema parsers.
+ * Provider capability coverage is what's left to pin here.
  */
 
 import { test, expect } from 'bun:test';
-import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing';
 import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
 
-test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
-  const cost = estimateCostUsd({ input: 1000, output: 500 }, 'unknown-model-7b');
-  expect(cost).toBe(0);
-});
-
-test('estimateCostUsd computes correctly for known Claude model', () => {
-  // claude-opus-4-7: $15/MTok input, $75/MTok output
-  // 1M input + 0.5M output = $15 + $37.50 = $52.50
-  const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7');
-  expect(cost).toBeCloseTo(52.50, 2);
-});
-
-test('estimateCostUsd applies cached input discount alongside uncached input', () => {
-  // tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%.
-  // 0 uncached input, 1M cached → 10% of 15 = $1.50
-  const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7');
-  expect(cost1).toBeCloseTo(1.50, 2);
-  // 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25
-  const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7');
-  expect(cost2).toBeCloseTo(8.25, 2);
-});
-
-test('PRICING table covers the key model families', () => {
-  expect(PRICING['claude-opus-4-7']).toBeDefined();
-  expect(PRICING['claude-sonnet-4-6']).toBeDefined();
-  expect(PRICING['gpt-5.4']).toBeDefined();
-  expect(PRICING['gemini-2.5-pro']).toBeDefined();
-});
-
 test('missingTools reports unsupported tools per provider', () => {
   // GPT/Codex doesn't expose Edit, Glob, Grep
   expect(missingTools('gpt', ['Edit', 'Glob', 'Grep'])).toEqual(['Edit', 'Glob', 'Grep']);

test/gemini-benchmark-adapter.test.ts

@@ -1,47 +0,0 @@
-import { describe, expect, test } from 'bun:test';
-import { parseGeminiStreamJson } from '../lib/model-benchmark/providers/gemini';
-
-describe('parseGeminiStreamJson', () => {
-  test('parses the current Gemini CLI content/stats schema', () => {
-    const raw = [
-      JSON.stringify({ type: 'init', model: 'auto' }),
-      JSON.stringify({ type: 'message', role: 'user', content: 'ignore me' }),
-      JSON.stringify({ type: 'message', role: 'assistant', content: 'real output' }),
-      JSON.stringify({ type: 'tool_use', tool_name: 'read_file' }),
-      JSON.stringify({
-        type: 'result',
-        stats: {
-          input_tokens: 101,
-          output_tokens: 17,
-          models: { 'gemini-3.1-flash-lite': { api: { totalRequests: 1 } } },
-        },
-      }),
-    ].join('\n');
-
-    expect(parseGeminiStreamJson(raw)).toEqual({
-      output: 'real output',
-      tokens: { input: 101, output: 17 },
-      toolCalls: 1,
-      modelUsed: 'gemini-3.1-flash-lite',
-    });
-  });
-
-  test('retains compatibility with legacy text/usage fields', () => {
-    const raw = [
-      JSON.stringify({ type: 'init', model: 'gemini-2.5-pro' }),
-      JSON.stringify({ type: 'message', role: 'assistant', text: 'legacy output' }),
-      JSON.stringify({
-        type: 'result',
-        usage: { input_token_count: 23, output_token_count: 5 },
-      }),
-      '{malformed',
-    ].join('\n');
-
-    expect(parseGeminiStreamJson(raw)).toEqual({
-      output: 'legacy output',
-      tokens: { input: 23, output: 5 },
-      toolCalls: 0,
-      modelUsed: 'gemini-2.5-pro',
-    });
-  });
-});

test/helpers/pricing.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/pricing';

test/skill-e2e-benchmark-providers.test.ts

@@ -7,15 +7,12 @@
  * to keep cost near $0.001/provider/run.
  *
  * What this catches that unit tests don't:
- *   - CLI output-format drift (the #1 silent breakage path)
- *   - Token parsing from real provider responses
+ *   - CLI invocation drift (a flag rename or trust-prompt change breaking a run)
  *   - Auth-failure vs timeout vs rate-limit error code routing
- *   - Cost estimation on real token counts
- *   - Parallel execution via Promise.allSettled — slow provider doesn't block fast
+ *   - The adapter terminates without throwing and returns plain-text output
  *
  * NOT covered here (would need dedicated test files):
  *   - Quality judge integration (autoevals ClosedQA, opt-in)
- *   - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0`
  */
 
 import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
@@ -91,13 +88,9 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
       throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`);
     }
     expect(result.output.toLowerCase()).toContain('ok');
-    expect(result.tokens.input).toBeGreaterThan(0);
-    expect(result.tokens.output).toBeGreaterThan(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
     expect(result.modelUsed.length).toBeGreaterThan(0);
-    const cost = claude.estimateCost(result.tokens, result.modelUsed);
-    expect(cost).toBeGreaterThan(0);
   }, 150_000);
 
   test('gpt: trivial prompt produces parseable output', async () => {
@@ -111,12 +104,8 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
       throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`);
     }
     expect(result.output.toLowerCase()).toContain('ok');
-    expect(result.tokens.input).toBeGreaterThan(0);
-    expect(result.tokens.output).toBeGreaterThan(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
-    const cost = gpt.estimateCost(result.tokens, result.modelUsed);
-    expect(cost).toBeGreaterThan(0);
   }, 150_000);
 
   test('gemini: trivial prompt produces parseable output', async () => {
@@ -129,17 +118,10 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     if (result.error) {
       throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`);
     }
-    // Gemini CLI occasionally returns empty output even on successful runs
-    // (model returned content the CLI parser missed, intermittent stream issues).
-    // We assert the adapter ran end-to-end without erroring and reports a non-
-    // empty token count instead of grepping the literal "ok" — that string
-    // assertion was too brittle for a smoke that's really about "did the
-    // adapter wire up and the run terminate successfully?"
+    // Gemini CLI can return empty output on otherwise-successful runs in some
+    // environments. This smoke is about "did the adapter wire up and terminate
+    // without throwing" — assert the shape, not the content.
     expect(typeof result.output).toBe('string');
-    // Gemini CLI sometimes returns 0 tokens in the result event (older responses);
-    // assert non-negative instead of strictly positive.
-    expect(result.tokens.input).toBeGreaterThanOrEqual(0);
-    expect(result.tokens.output).toBeGreaterThanOrEqual(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
   }, 150_000);