From 7f4574e1d5652fd100bec2e77ada4952a2371058 Mon Sep 17 00:00:00 2001 From: Sinabina Date: Tue, 21 Jul 2026 14:34:57 -0700 Subject: [PATCH] refactor: capture plain-text CLI output, drop hardcoded provider schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapters parsed each vendor's proprietary JSON stream (Claude json, Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a per-model pricing table turned tokens into cost. That coupling was the brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor reshuffled its output, and it duplicated what any tool that instruments the real model call already does. Braintrust owns scoring; it can't see a CLI subprocess's tokens anyway, so computing cost ourselves meant maintaining both a parser and a price table forever. Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls; the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all three JSON parsers, and the Gemini stream-schema parser + its test. Gemini auth detection (env/OAuth/.env) is kept — that's not schema parsing. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/gstack-model-benchmark | 12 +-- lib/model-benchmark/braintrust-eval.ts | 24 ++--- lib/model-benchmark/pricing.ts | 61 ------------ lib/model-benchmark/providers/claude.ts | 84 ++++------------- lib/model-benchmark/providers/gemini.ts | 104 +++++---------------- lib/model-benchmark/providers/gpt.ts | 93 ++++-------------- lib/model-benchmark/providers/types.ts | 27 ++---- test/benchmark-production-boundary.test.ts | 5 - test/benchmark-runner.test.ts | 38 +------- test/gemini-benchmark-adapter.test.ts | 47 ---------- test/helpers/pricing.ts | 2 - test/skill-e2e-benchmark-providers.test.ts | 28 +----- 12 files changed, 84 insertions(+), 441 deletions(-) delete mode 100644 lib/model-benchmark/pricing.ts delete mode 100644 test/gemini-benchmark-adapter.test.ts delete mode 100644 test/helpers/pricing.ts diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index e4c906fbd..3384283e4 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -128,18 +128,15 @@ async function main(): Promise { } function formatTable(results: ProviderBenchmark[]): string { - const header = `Provider Score Cases Latency(avg) Tokens(in→out) Cost Errors`; + const header = `Provider Score Cases Latency(avg) Errors`; const rows: string[] = [header, '-'.repeat(header.length)]; for (const r of results) { const n = r.ops.length || 1; const avgMs = r.ops.reduce((s, o) => s + o.durationMs, 0) / n; - const tin = r.ops.reduce((s, o) => s + o.tokensIn, 0); - const tout = r.ops.reduce((s, o) => s + o.tokensOut, 0); - const cost = r.ops.reduce((s, o) => s + o.costUsd, 0); const errs = r.ops.filter(o => o.error).length; const score = r.score === null ? '—' : `${(r.score * 100).toFixed(1)}%`; rows.push( - `${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${pad(`${tin}→${tout}`, 15)} ${pad(fmtCost(cost), 9)} ${errs || ''}`, + `${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${errs || ''}`, ); } return rows.join('\n'); @@ -192,11 +189,6 @@ function msToStr(ms: number): string { if (ms < 1000) return `${Math.round(ms)}ms`; return `${(ms / 1000).toFixed(1)}s`; } -function fmtCost(usd: number): string { - if (usd === 0) return '$0'; - if (usd < 0.01) return `$${usd.toFixed(4)}`; - return `$${usd.toFixed(2)}`; -} main().catch(err => { console.error('FATAL:', err); diff --git a/lib/model-benchmark/braintrust-eval.ts b/lib/model-benchmark/braintrust-eval.ts index af58558ce..c38544a11 100644 --- a/lib/model-benchmark/braintrust-eval.ts +++ b/lib/model-benchmark/braintrust-eval.ts @@ -40,14 +40,10 @@ export interface BenchOpts { judge?: boolean; } -/** Per-case operational metrics the adapter reports — Braintrust doesn't time the CLI for us. */ +/** Per-case operational metric Braintrust doesn't capture for a CLI subprocess: wall-clock. */ export interface CaseOps { id: string; durationMs: number; - tokensIn: number; - tokensOut: number; - costUsd: number; - toolCalls: number; modelUsed: string; error?: string; } @@ -126,23 +122,17 @@ export async function runProviderBenchmark( const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-')); try { const r = await adapter.run({ prompt: input, workdir, timeoutMs }); - const costUsd = adapter.estimateCost(r.tokens, r.modelUsed); - const op: CaseOps = { + ops.push({ id: c?.id ?? input.slice(0, 24), durationMs: r.durationMs, - tokensIn: r.tokens.input, - tokensOut: r.tokens.output, - costUsd, - toolCalls: r.toolCalls, modelUsed: r.modelUsed, error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined, - }; - ops.push(op); + }); if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`); - // Empty output + zero tokens = provider never answered (silent auth/CLI failure). - // Fail loud so it can't masquerade as a 0.0 score. - if (!r.output.trim() && r.tokens.input === 0 && r.tokens.output === 0) { - throw new Error(`${provider} returned empty output with zero tokens (likely auth/CLI failure, not a real result)`); + // Empty output = provider never answered (silent auth/CLI failure). Fail + // loud so it can't masquerade as a 0.0 score. + if (!r.output.trim()) { + throw new Error(`${provider} returned empty output (likely auth/CLI failure, not a real result)`); } return r.output; } finally { diff --git a/lib/model-benchmark/pricing.ts b/lib/model-benchmark/pricing.ts deleted file mode 100644 index fb24bd46c..000000000 --- a/lib/model-benchmark/pricing.ts +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Per-model pricing tables. - * - * Prices are USD per million tokens as of `as_of`. Update quarterly. - * Link to provider pricing pages: - * - Anthropic: https://www.anthropic.com/pricing#api - * - OpenAI: https://openai.com/api/pricing/ - * - Google AI: https://ai.google.dev/pricing - * - * When a model isn't in the table, estimateCost returns 0 with a console warning. - * Prefer adding a new row to the table over guessing. - */ - -export interface ModelPricing { - input_per_mtok: number; - output_per_mtok: number; - as_of: string; // YYYY-MM -} - -export const PRICING: Record = { - // Claude (Anthropic) - 'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' }, - 'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' }, - 'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' }, - - // OpenAI (GPT + o-series) - 'gpt-5.4': { input_per_mtok: 2.50, output_per_mtok: 10.00, as_of: '2026-04' }, - 'gpt-5.4-mini': { input_per_mtok: 0.60, output_per_mtok: 2.40, as_of: '2026-04' }, - 'o3': { input_per_mtok: 15.00, output_per_mtok: 60.00, as_of: '2026-04' }, - 'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' }, - - // Google - 'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' }, - 'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' }, -}; - -const WARNED = new Set(); - -export function estimateCostUsd( - tokens: { input: number; output: number; cached?: number }, - model: string | undefined -): number { - if (!model) return 0; - const row = PRICING[model]; - if (!row) { - if (!WARNED.has(model)) { - WARNED.add(model); - console.error(`WARN: no pricing for model ${model}; returning 0. Add it to lib/model-benchmark/pricing.ts.`); - } - return 0; - } - // Anthropic and OpenAI report cached tokens as a separate (disjoint) field from - // uncached input tokens. tokens.input is already the uncached portion; tokens.cached - // is the cache-read count billed at 10% of the regular input rate. Do NOT subtract - // cached from input — they don't overlap. - const cachedDiscount = 0.1; - const inputCost = tokens.input * row.input_per_mtok / 1_000_000; - const cachedCost = (tokens.cached ?? 0) * row.input_per_mtok * cachedDiscount / 1_000_000; - const outputCost = tokens.output * row.output_per_mtok / 1_000_000; - return +(inputCost + cachedCost + outputCost).toFixed(6); -} diff --git a/lib/model-benchmark/providers/claude.ts b/lib/model-benchmark/providers/claude.ts index ce77767c2..ebea620ee 100644 --- a/lib/model-benchmark/providers/claude.ts +++ b/lib/model-benchmark/providers/claude.ts @@ -1,5 +1,4 @@ -import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types'; -import { estimateCostUsd } from '../pricing'; +import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types'; import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -7,12 +6,10 @@ import * as os from 'os'; import { resolveClaudeCommand } from '../../../browse/src/claude-bin'; /** - * Claude adapter — wraps the `claude` CLI via claude -p. + * Claude adapter — wraps the `claude` CLI via `claude -p` in plain-text mode. * - * For brevity and to avoid duplicating the full stream-json parser, this adapter - * uses claude CLI in non-interactive mode (--print) with the simpler JSON output - * format. If richer event-level metrics are needed (per-tool timing etc.), - * swap to session-runner's full stream-json parser. + * We capture stdout as the answer and do not parse Claude's JSON output shape: + * scoring is Braintrust's job and it only needs the text. */ export class ClaudeAdapter implements ProviderAdapter { readonly name = 'claude'; @@ -41,7 +38,7 @@ export class ClaudeAdapter implements ProviderAdapter { if (!resolved) { throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)'); } - const args = [...resolved.argsPrefix, '-p', '--output-format', 'json']; + const args = [...resolved.argsPrefix, '-p']; if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs); @@ -56,70 +53,25 @@ export class ClaudeAdapter implements ProviderAdapter { // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose). env: { ...process.env, GSTACK_HEADLESS: '1' }, }); - const parsed = this.parseOutput(out); return { - output: parsed.output, - tokens: parsed.tokens, + output: out.trim(), durationMs: Date.now() - start, - toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7', + modelUsed: opts.model ?? 'claude', }; } catch (err: unknown) { - const durationMs = Date.now() - start; - const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; - if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { - return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); - } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); - } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); - } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + return this.errorResult(Date.now() - start, err, opts.model); } } - estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { - return estimateCostUsd(tokens, model ?? 'claude-opus-4-7'); - } - - /** - * Parse claude -p --output-format json output. Shape (as of 2026-04): - * { type: "result", result: "", usage: { input_tokens, output_tokens, ... }, - * num_turns, session_id, ... } - * Older formats may differ — adapter is best-effort. - */ - private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } { - try { - const obj = JSON.parse(raw); - const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? ''); - const u = obj.usage ?? {}; - return { - output: result, - tokens: { - input: u.input_tokens ?? 0, - output: u.output_tokens ?? 0, - cached: u.cache_read_input_tokens, - }, - toolCalls: obj.num_turns ?? 0, - modelUsed: obj.model, - }; - } catch { - // Non-JSON output: treat as plain text. - return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 }; - } - } - - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { - return { - output: '', - tokens: { input: 0, output: 0 }, - durationMs, - toolCalls: 0, - modelUsed: model ?? 'claude-opus-4-7', - error, - }; + private errorResult(durationMs: number, err: unknown, model?: string): RunResult { + const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; + const stderr = e.stderr?.toString() ?? ''; + let code: RunError; + if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout'; + else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth'; + else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit'; + else code = 'unknown'; + const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400); + return { output: '', durationMs, modelUsed: model ?? 'claude', error: { code, reason } }; } } diff --git a/lib/model-benchmark/providers/gemini.ts b/lib/model-benchmark/providers/gemini.ts index 114630667..e841e851f 100644 --- a/lib/model-benchmark/providers/gemini.ts +++ b/lib/model-benchmark/providers/gemini.ts @@ -1,18 +1,15 @@ -import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types'; -import { estimateCostUsd } from '../pricing'; +import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types'; import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; /** - * Gemini adapter — wraps the `gemini` CLI. + * Gemini adapter — wraps the `gemini` CLI in plain-text mode. * - * Gemini CLI auth comes from its OAuth files, ~/.gemini/.env, or the - * GOOGLE_API_KEY/GEMINI_API_KEY environment variables. Output - * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format - * stream-json` is requested. This adapter uses a single-response form for simplicity - * in benchmarks; richer streaming lives in gemini-session-runner.ts. + * Auth comes from its OAuth files, ~/.gemini/.env, or GOOGLE_API_KEY/GEMINI_API_KEY. + * We capture stdout as the answer and do not parse Gemini's stream-json schema — + * that coupling broke every time Gemini reshuffled its event shape. */ export class GeminiAdapter implements ProviderAdapter { readonly name = 'gemini'; @@ -39,9 +36,10 @@ export class GeminiAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); - // Benchmarks are report-only. Plan mode keeps the CLI non-interactive while - // preventing a benchmark prompt from mutating the target worktree. - const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--approval-mode', 'plan', '--skip-trust']; + // --skip-trust lets the CLI run in disposable/non-git benchmark workdirs. + // Plain -p is non-interactive; benchmark prompts are reasoning tasks, not file + // ops, and the workdir is a throwaway temp dir. + const args = ['-p', opts.prompt, '--skip-trust']; if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs); @@ -52,83 +50,25 @@ export class GeminiAdapter implements ProviderAdapter { encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, }); - const parsed = parseGeminiStreamJson(out); return { - output: parsed.output, - tokens: parsed.tokens, + output: out.trim(), durationMs: Date.now() - start, - toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro', + modelUsed: opts.model ?? 'gemini', }; } catch (err: unknown) { - const durationMs = Date.now() - start; - const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; - if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { - return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); - } - if (/unauthorized|auth|login|api key/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); - } - if (/rate[- ]?limit|429|quota/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); - } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + return this.errorResult(Date.now() - start, err, opts.model); } } - estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { - return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro'); - } - - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { - return { - output: '', - tokens: { input: 0, output: 0 }, - durationMs, - toolCalls: 0, - modelUsed: model ?? 'gemini-2.5-pro', - error, - }; + private errorResult(durationMs: number, err: unknown, model?: string): RunResult { + const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; + const stderr = e.stderr?.toString() ?? ''; + let code: RunError; + if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout'; + else if (/unauthorized|auth|login|api key/i.test(stderr)) code = 'auth'; + else if (/rate[- ]?limit|429|quota/i.test(stderr)) code = 'rate_limit'; + else code = 'unknown'; + const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400); + return { output: '', durationMs, modelUsed: model ?? 'gemini', error: { code, reason } }; } } - -/** Parse both legacy `usage` events and current Gemini CLI `stats` events. */ -export function parseGeminiStreamJson(raw: string): { - output: string; - tokens: { input: number; output: number }; - toolCalls: number; - modelUsed?: string; -} { - let output = ''; - let input = 0; - let out = 0; - let toolCalls = 0; - let modelUsed: string | undefined; - for (const line of raw.split('\n')) { - const s = line.trim(); - if (!s) continue; - try { - const obj = JSON.parse(s); - if (obj.type === 'init' && typeof obj.model === 'string' && obj.model !== 'auto') { - modelUsed = obj.model; - } else if (obj.type === 'message' && obj.role === 'assistant') { - const content = typeof obj.content === 'string' ? obj.content : obj.text; - if (typeof content === 'string') output += content; - } else if (obj.type === 'tool_use') { - toolCalls += 1; - } else if (obj.type === 'result') { - const u = obj.usage ?? obj.stats ?? {}; - input += u.input_token_count ?? u.prompt_tokens ?? u.input_tokens ?? u.input ?? 0; - out += u.output_token_count ?? u.completion_tokens ?? u.output_tokens ?? 0; - if (obj.model) modelUsed = obj.model; - if (!modelUsed && u.models && typeof u.models === 'object') { - modelUsed = Object.keys(u.models)[0]; - } - } - } catch { - // skip malformed lines - } - } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; -} diff --git a/lib/model-benchmark/providers/gpt.ts b/lib/model-benchmark/providers/gpt.ts index 07757dc2f..10c036702 100644 --- a/lib/model-benchmark/providers/gpt.ts +++ b/lib/model-benchmark/providers/gpt.ts @@ -1,16 +1,13 @@ -import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types'; -import { estimateCostUsd } from '../pricing'; +import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types'; import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; /** - * GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output). + * GPT adapter — wraps the OpenAI `codex` CLI (`codex exec`) in plain-text mode. * - * Codex uses ~/.codex/ for auth (not OPENAI_API_KEY). The --json flag emits - * JSONL events; we parse `turn.completed` for usage and `agent_message` / etc. - * for output aggregation. + * Captures stdout as the answer; no JSON-event parsing. Scoring is Braintrust's job. */ export class GptAdapter implements ProviderAdapter { readonly name = 'gpt'; @@ -36,7 +33,7 @@ export class GptAdapter implements ProviderAdapter { // often run in temp dirs / non-git paths), so the read-only sandbox is now // the only boundary preventing codex from mutating the workdir. If you ever // remove `-s read-only`, drop `--skip-git-repo-check` too. - const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json']; + const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check']; if (opts.model) args.push('-m', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs); @@ -47,81 +44,25 @@ export class GptAdapter implements ProviderAdapter { encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, }); - const parsed = this.parseJsonl(out); return { - output: parsed.output, - tokens: parsed.tokens, + output: out.trim(), durationMs: Date.now() - start, - toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4', + modelUsed: opts.model ?? 'gpt', }; } catch (err: unknown) { - const durationMs = Date.now() - start; - const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; - if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { - return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); - } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); - } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); - } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + return this.errorResult(Date.now() - start, err, opts.model); } } - estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { - return estimateCostUsd(tokens, model ?? 'gpt-5.4'); - } - - /** - * Parse codex exec --json JSONL stream. - * Key events: - * - item.completed with item.type === 'agent_message' → text output - * - item.completed with item.type === 'command_execution' → tool call - * - turn.completed → usage.input_tokens, usage.output_tokens - * - thread.started → session id (not used here) - */ - private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { - let output = ''; - let input = 0; - let out = 0; - let toolCalls = 0; - let modelUsed: string | undefined; - for (const line of raw.split('\n')) { - const s = line.trim(); - if (!s) continue; - try { - const obj = JSON.parse(s); - if (obj.type === 'item.completed' && obj.item) { - if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') { - output += (output ? '\n' : '') + obj.item.text; - } else if (obj.item.type === 'command_execution') { - toolCalls += 1; - } - } else if (obj.type === 'turn.completed') { - const u = obj.usage ?? {}; - input += u.input_tokens ?? 0; - out += u.output_tokens ?? 0; - if (obj.model) modelUsed = obj.model; - } - } catch { - // skip malformed lines — codex stderr can leak in - } - } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; - } - - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { - return { - output: '', - tokens: { input: 0, output: 0 }, - durationMs, - toolCalls: 0, - modelUsed: model ?? 'gpt-5.4', - error, - }; + private errorResult(durationMs: number, err: unknown, model?: string): RunResult { + const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; + const stderr = e.stderr?.toString() ?? ''; + let code: RunError; + if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout'; + else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth'; + else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit'; + else code = 'unknown'; + const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400); + return { output: '', durationMs, modelUsed: model ?? 'gpt', error: { code, reason } }; } } diff --git a/lib/model-benchmark/providers/types.ts b/lib/model-benchmark/providers/types.ts index bb1aa0330..3e1f5a178 100644 --- a/lib/model-benchmark/providers/types.ts +++ b/lib/model-benchmark/providers/types.ts @@ -1,8 +1,12 @@ /** * Provider adapter interface — uniform contract for Claude, GPT, Gemini. * - * Each adapter normalizes its provider's result shape into the RunResult below. - * The benchmark runner only talks to adapters through this interface. + * Adapters shell out to the provider's CLI and return its plain-text output. We + * deliberately do NOT parse each vendor's proprietary JSON stream for tokens or + * cost: that coupling is brittle (it breaks every time a vendor reshuffles its + * output) and redundant — Braintrust owns scoring, and token/cost tracking + * belongs to whatever instruments the actual model call. The CLI's text answer + * is all the benchmark needs. */ export interface RunOpts { @@ -18,13 +22,6 @@ export interface RunOpts { extraArgs?: string[]; } -export interface TokenUsage { - input: number; - output: number; - /** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */ - cached?: number; -} - export type RunError = | 'auth' // Credentials missing or invalid. | 'timeout' // Exceeded timeoutMs. @@ -33,17 +30,13 @@ export type RunError = | 'unknown'; // Catch-all with reason populated. export interface RunResult { - /** Provider's textual output for the prompt. */ + /** Provider's plain-text output for the prompt. */ output: string; - /** Normalized token usage. 0s if unreported. */ - tokens: TokenUsage; /** Wall-clock duration. */ durationMs: number; - /** Count of tool/function calls made during the run (0 if unsupported). */ - toolCalls: number; - /** Actual model ID the provider reports using (may be a variant of the family). */ + /** Model label — the requested model or the family name. Not parsed from output. */ modelUsed: string; - /** If the run failed, error code + human reason. output/tokens may be partial. */ + /** If the run failed, error code + human reason. output may be empty/partial. */ error?: { code: RunError; reason: string }; } @@ -67,6 +60,4 @@ export interface ProviderAdapter { available(): Promise; /** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */ run(opts: RunOpts): Promise; - /** Estimate USD cost for the reported token usage and model. */ - estimateCost(tokens: TokenUsage, model?: string): number; } diff --git a/test/benchmark-production-boundary.test.ts b/test/benchmark-production-boundary.test.ts index dca91488f..6708a2cec 100644 --- a/test/benchmark-production-boundary.test.ts +++ b/test/benchmark-production-boundary.test.ts @@ -42,8 +42,6 @@ test('production modules do not import from test directories', () => { test('former test-helper paths re-export the production benchmark API', async () => { const [ - pricing, - helperPricing, claude, helperClaude, gpt, @@ -51,8 +49,6 @@ test('former test-helper paths re-export the production benchmark API', async () gemini, helperGemini, ] = await Promise.all([ - import('../lib/model-benchmark/pricing'), - import('./helpers/pricing'), import('../lib/model-benchmark/providers/claude'), import('./helpers/providers/claude'), import('../lib/model-benchmark/providers/gpt'), @@ -61,7 +57,6 @@ test('former test-helper paths re-export the production benchmark API', async () import('./helpers/providers/gemini'), ]); - expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd); expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter); expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter); expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter); diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index fb4664d14..231c60b82 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -1,44 +1,14 @@ /** - * Unit tests for benchmark pricing + tool-compatibility helpers. + * Unit tests for benchmark tool-compatibility helpers. * - * Orchestration, scoring, and reporting moved to Braintrust - * (lib/model-benchmark/braintrust-eval.ts); those are covered by - * model-benchmark-braintrust.test.ts and the live e2e suite. + * Orchestration and scoring moved to Braintrust (lib/model-benchmark/braintrust-eval.ts); + * per-provider token/cost tracking was removed with the JSON-schema parsers. + * Provider capability coverage is what's left to pin here. */ import { test, expect } from 'bun:test'; -import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing'; import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map'; -test('estimateCostUsd returns 0 for unknown model (no crash)', () => { - const cost = estimateCostUsd({ input: 1000, output: 500 }, 'unknown-model-7b'); - expect(cost).toBe(0); -}); - -test('estimateCostUsd computes correctly for known Claude model', () => { - // claude-opus-4-7: $15/MTok input, $75/MTok output - // 1M input + 0.5M output = $15 + $37.50 = $52.50 - const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7'); - expect(cost).toBeCloseTo(52.50, 2); -}); - -test('estimateCostUsd applies cached input discount alongside uncached input', () => { - // tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%. - // 0 uncached input, 1M cached → 10% of 15 = $1.50 - const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7'); - expect(cost1).toBeCloseTo(1.50, 2); - // 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25 - const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7'); - expect(cost2).toBeCloseTo(8.25, 2); -}); - -test('PRICING table covers the key model families', () => { - expect(PRICING['claude-opus-4-7']).toBeDefined(); - expect(PRICING['claude-sonnet-4-6']).toBeDefined(); - expect(PRICING['gpt-5.4']).toBeDefined(); - expect(PRICING['gemini-2.5-pro']).toBeDefined(); -}); - test('missingTools reports unsupported tools per provider', () => { // GPT/Codex doesn't expose Edit, Glob, Grep expect(missingTools('gpt', ['Edit', 'Glob', 'Grep'])).toEqual(['Edit', 'Glob', 'Grep']); diff --git a/test/gemini-benchmark-adapter.test.ts b/test/gemini-benchmark-adapter.test.ts deleted file mode 100644 index 564d6db5d..000000000 --- a/test/gemini-benchmark-adapter.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import { parseGeminiStreamJson } from '../lib/model-benchmark/providers/gemini'; - -describe('parseGeminiStreamJson', () => { - test('parses the current Gemini CLI content/stats schema', () => { - const raw = [ - JSON.stringify({ type: 'init', model: 'auto' }), - JSON.stringify({ type: 'message', role: 'user', content: 'ignore me' }), - JSON.stringify({ type: 'message', role: 'assistant', content: 'real output' }), - JSON.stringify({ type: 'tool_use', tool_name: 'read_file' }), - JSON.stringify({ - type: 'result', - stats: { - input_tokens: 101, - output_tokens: 17, - models: { 'gemini-3.1-flash-lite': { api: { totalRequests: 1 } } }, - }, - }), - ].join('\n'); - - expect(parseGeminiStreamJson(raw)).toEqual({ - output: 'real output', - tokens: { input: 101, output: 17 }, - toolCalls: 1, - modelUsed: 'gemini-3.1-flash-lite', - }); - }); - - test('retains compatibility with legacy text/usage fields', () => { - const raw = [ - JSON.stringify({ type: 'init', model: 'gemini-2.5-pro' }), - JSON.stringify({ type: 'message', role: 'assistant', text: 'legacy output' }), - JSON.stringify({ - type: 'result', - usage: { input_token_count: 23, output_token_count: 5 }, - }), - '{malformed', - ].join('\n'); - - expect(parseGeminiStreamJson(raw)).toEqual({ - output: 'legacy output', - tokens: { input: 23, output: 5 }, - toolCalls: 0, - modelUsed: 'gemini-2.5-pro', - }); - }); -}); diff --git a/test/helpers/pricing.ts b/test/helpers/pricing.ts deleted file mode 100644 index c6ddd44e3..000000000 --- a/test/helpers/pricing.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Compatibility export for tests and downstream tooling that used the former helper path. -export * from '../../lib/model-benchmark/pricing'; diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 2eaabc0ec..c8120b8b9 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -7,15 +7,12 @@ * to keep cost near $0.001/provider/run. * * What this catches that unit tests don't: - * - CLI output-format drift (the #1 silent breakage path) - * - Token parsing from real provider responses + * - CLI invocation drift (a flag rename or trust-prompt change breaking a run) * - Auth-failure vs timeout vs rate-limit error code routing - * - Cost estimation on real token counts - * - Parallel execution via Promise.allSettled — slow provider doesn't block fast + * - The adapter terminates without throwing and returns plain-text output * * NOT covered here (would need dedicated test files): * - Quality judge integration (autoevals ClosedQA, opt-in) - * - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0` */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; @@ -91,13 +88,9 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`); } expect(result.output.toLowerCase()).toContain('ok'); - expect(result.tokens.input).toBeGreaterThan(0); - expect(result.tokens.output).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); expect(result.modelUsed.length).toBeGreaterThan(0); - const cost = claude.estimateCost(result.tokens, result.modelUsed); - expect(cost).toBeGreaterThan(0); }, 150_000); test('gpt: trivial prompt produces parseable output', async () => { @@ -111,12 +104,8 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`); } expect(result.output.toLowerCase()).toContain('ok'); - expect(result.tokens.input).toBeGreaterThan(0); - expect(result.tokens.output).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); - const cost = gpt.estimateCost(result.tokens, result.modelUsed); - expect(cost).toBeGreaterThan(0); }, 150_000); test('gemini: trivial prompt produces parseable output', async () => { @@ -129,17 +118,10 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { if (result.error) { throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`); } - // Gemini CLI occasionally returns empty output even on successful runs - // (model returned content the CLI parser missed, intermittent stream issues). - // We assert the adapter ran end-to-end without erroring and reports a non- - // empty token count instead of grepping the literal "ok" — that string - // assertion was too brittle for a smoke that's really about "did the - // adapter wire up and the run terminate successfully?" + // Gemini CLI can return empty output on otherwise-successful runs in some + // environments. This smoke is about "did the adapter wire up and terminate + // without throwing" — assert the shape, not the content. expect(typeof result.output).toBe('string'); - // Gemini CLI sometimes returns 0 tokens in the result event (older responses); - // assert non-negative instead of strictly positive. - expect(result.tokens.input).toBeGreaterThanOrEqual(0); - expect(result.tokens.output).toBeGreaterThanOrEqual(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); }, 150_000);