mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 04:10:47 +02:00
refactor: capture plain-text CLI output, drop hardcoded provider schemas
The adapters parsed each vendor's proprietary JSON stream (Claude json, Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a per-model pricing table turned tokens into cost. That coupling was the brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor reshuffled its output, and it duplicated what any tool that instruments the real model call already does. Braintrust owns scoring; it can't see a CLI subprocess's tokens anyway, so computing cost ourselves meant maintaining both a parser and a price table forever. Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls; the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all three JSON parsers, and the Gemini stream-schema parser + its test. Gemini auth detection (env/OAuth/.env) is kept — that's not schema parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sinabina
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
36f972f2e4
commit
7f4574e1d5
@@ -40,14 +40,10 @@ export interface BenchOpts {
|
||||
judge?: boolean;
|
||||
}
|
||||
|
||||
/** Per-case operational metrics the adapter reports — Braintrust doesn't time the CLI for us. */
|
||||
/** Per-case operational metric Braintrust doesn't capture for a CLI subprocess: wall-clock. */
|
||||
export interface CaseOps {
|
||||
id: string;
|
||||
durationMs: number;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
costUsd: number;
|
||||
toolCalls: number;
|
||||
modelUsed: string;
|
||||
error?: string;
|
||||
}
|
||||
@@ -126,23 +122,17 @@ export async function runProviderBenchmark(
|
||||
const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-'));
|
||||
try {
|
||||
const r = await adapter.run({ prompt: input, workdir, timeoutMs });
|
||||
const costUsd = adapter.estimateCost(r.tokens, r.modelUsed);
|
||||
const op: CaseOps = {
|
||||
ops.push({
|
||||
id: c?.id ?? input.slice(0, 24),
|
||||
durationMs: r.durationMs,
|
||||
tokensIn: r.tokens.input,
|
||||
tokensOut: r.tokens.output,
|
||||
costUsd,
|
||||
toolCalls: r.toolCalls,
|
||||
modelUsed: r.modelUsed,
|
||||
error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined,
|
||||
};
|
||||
ops.push(op);
|
||||
});
|
||||
if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`);
|
||||
// Empty output + zero tokens = provider never answered (silent auth/CLI failure).
|
||||
// Fail loud so it can't masquerade as a 0.0 score.
|
||||
if (!r.output.trim() && r.tokens.input === 0 && r.tokens.output === 0) {
|
||||
throw new Error(`${provider} returned empty output with zero tokens (likely auth/CLI failure, not a real result)`);
|
||||
// Empty output = provider never answered (silent auth/CLI failure). Fail
|
||||
// loud so it can't masquerade as a 0.0 score.
|
||||
if (!r.output.trim()) {
|
||||
throw new Error(`${provider} returned empty output (likely auth/CLI failure, not a real result)`);
|
||||
}
|
||||
return r.output;
|
||||
} finally {
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
/**
|
||||
* Per-model pricing tables.
|
||||
*
|
||||
* Prices are USD per million tokens as of `as_of`. Update quarterly.
|
||||
* Link to provider pricing pages:
|
||||
* - Anthropic: https://www.anthropic.com/pricing#api
|
||||
* - OpenAI: https://openai.com/api/pricing/
|
||||
* - Google AI: https://ai.google.dev/pricing
|
||||
*
|
||||
* When a model isn't in the table, estimateCost returns 0 with a console warning.
|
||||
* Prefer adding a new row to the table over guessing.
|
||||
*/
|
||||
|
||||
export interface ModelPricing {
|
||||
input_per_mtok: number;
|
||||
output_per_mtok: number;
|
||||
as_of: string; // YYYY-MM
|
||||
}
|
||||
|
||||
export const PRICING: Record<string, ModelPricing> = {
|
||||
// Claude (Anthropic)
|
||||
'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' },
|
||||
'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' },
|
||||
'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
|
||||
// OpenAI (GPT + o-series)
|
||||
'gpt-5.4': { input_per_mtok: 2.50, output_per_mtok: 10.00, as_of: '2026-04' },
|
||||
'gpt-5.4-mini': { input_per_mtok: 0.60, output_per_mtok: 2.40, as_of: '2026-04' },
|
||||
'o3': { input_per_mtok: 15.00, output_per_mtok: 60.00, as_of: '2026-04' },
|
||||
'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' },
|
||||
|
||||
// Google
|
||||
'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' },
|
||||
};
|
||||
|
||||
const WARNED = new Set<string>();
|
||||
|
||||
export function estimateCostUsd(
|
||||
tokens: { input: number; output: number; cached?: number },
|
||||
model: string | undefined
|
||||
): number {
|
||||
if (!model) return 0;
|
||||
const row = PRICING[model];
|
||||
if (!row) {
|
||||
if (!WARNED.has(model)) {
|
||||
WARNED.add(model);
|
||||
console.error(`WARN: no pricing for model ${model}; returning 0. Add it to lib/model-benchmark/pricing.ts.`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
// Anthropic and OpenAI report cached tokens as a separate (disjoint) field from
|
||||
// uncached input tokens. tokens.input is already the uncached portion; tokens.cached
|
||||
// is the cache-read count billed at 10% of the regular input rate. Do NOT subtract
|
||||
// cached from input — they don't overlap.
|
||||
const cachedDiscount = 0.1;
|
||||
const inputCost = tokens.input * row.input_per_mtok / 1_000_000;
|
||||
const cachedCost = (tokens.cached ?? 0) * row.input_per_mtok * cachedDiscount / 1_000_000;
|
||||
const outputCost = tokens.output * row.output_per_mtok / 1_000_000;
|
||||
return +(inputCost + cachedCost + outputCost).toFixed(6);
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -7,12 +6,10 @@ import * as os from 'os';
|
||||
import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
|
||||
|
||||
/**
|
||||
* Claude adapter — wraps the `claude` CLI via claude -p.
|
||||
* Claude adapter — wraps the `claude` CLI via `claude -p` in plain-text mode.
|
||||
*
|
||||
* For brevity and to avoid duplicating the full stream-json parser, this adapter
|
||||
* uses claude CLI in non-interactive mode (--print) with the simpler JSON output
|
||||
* format. If richer event-level metrics are needed (per-tool timing etc.),
|
||||
* swap to session-runner's full stream-json parser.
|
||||
* We capture stdout as the answer and do not parse Claude's JSON output shape:
|
||||
* scoring is Braintrust's job and it only needs the text.
|
||||
*/
|
||||
export class ClaudeAdapter implements ProviderAdapter {
|
||||
readonly name = 'claude';
|
||||
@@ -41,7 +38,7 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
if (!resolved) {
|
||||
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
|
||||
}
|
||||
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
|
||||
const args = [...resolved.argsPrefix, '-p'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -56,70 +53,25 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
||||
});
|
||||
const parsed = this.parseOutput(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
|
||||
modelUsed: opts.model ?? 'claude',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse claude -p --output-format json output. Shape (as of 2026-04):
|
||||
* { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
|
||||
* num_turns, session_id, ... }
|
||||
* Older formats may differ — adapter is best-effort.
|
||||
*/
|
||||
private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
|
||||
try {
|
||||
const obj = JSON.parse(raw);
|
||||
const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
|
||||
const u = obj.usage ?? {};
|
||||
return {
|
||||
output: result,
|
||||
tokens: {
|
||||
input: u.input_tokens ?? 0,
|
||||
output: u.output_tokens ?? 0,
|
||||
cached: u.cache_read_input_tokens,
|
||||
},
|
||||
toolCalls: obj.num_turns ?? 0,
|
||||
modelUsed: obj.model,
|
||||
};
|
||||
} catch {
|
||||
// Non-JSON output: treat as plain text.
|
||||
return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'claude-opus-4-7',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'claude', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
* Gemini adapter — wraps the `gemini` CLI in plain-text mode.
|
||||
*
|
||||
* Gemini CLI auth comes from its OAuth files, ~/.gemini/.env, or the
|
||||
* GOOGLE_API_KEY/GEMINI_API_KEY environment variables. Output
|
||||
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
|
||||
* stream-json` is requested. This adapter uses a single-response form for simplicity
|
||||
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
|
||||
* Auth comes from its OAuth files, ~/.gemini/.env, or GOOGLE_API_KEY/GEMINI_API_KEY.
|
||||
* We capture stdout as the answer and do not parse Gemini's stream-json schema —
|
||||
* that coupling broke every time Gemini reshuffled its event shape.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
readonly name = 'gemini';
|
||||
@@ -39,9 +36,10 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
// Benchmarks are report-only. Plan mode keeps the CLI non-interactive while
|
||||
// preventing a benchmark prompt from mutating the target worktree.
|
||||
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--approval-mode', 'plan', '--skip-trust'];
|
||||
// --skip-trust lets the CLI run in disposable/non-git benchmark workdirs.
|
||||
// Plain -p is non-interactive; benchmark prompts are reasoning tasks, not file
|
||||
// ops, and the workdir is a throwaway temp dir.
|
||||
const args = ['-p', opts.prompt, '--skip-trust'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -52,83 +50,25 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = parseGeminiStreamJson(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
|
||||
modelUsed: opts.model ?? 'gemini',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login|api key/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gemini-2.5-pro',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login|api key/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429|quota/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'gemini', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse both legacy `usage` events and current Gemini CLI `stats` events. */
|
||||
export function parseGeminiStreamJson(raw: string): {
|
||||
output: string;
|
||||
tokens: { input: number; output: number };
|
||||
toolCalls: number;
|
||||
modelUsed?: string;
|
||||
} {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'init' && typeof obj.model === 'string' && obj.model !== 'auto') {
|
||||
modelUsed = obj.model;
|
||||
} else if (obj.type === 'message' && obj.role === 'assistant') {
|
||||
const content = typeof obj.content === 'string' ? obj.content : obj.text;
|
||||
if (typeof content === 'string') output += content;
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? obj.stats ?? {};
|
||||
input += u.input_token_count ?? u.prompt_tokens ?? u.input_tokens ?? u.input ?? 0;
|
||||
out += u.output_token_count ?? u.completion_tokens ?? u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
if (!modelUsed && u.models && typeof u.models === 'object') {
|
||||
modelUsed = Object.keys(u.models)[0];
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (`codex exec`) in plain-text mode.
|
||||
*
|
||||
* Codex uses ~/.codex/ for auth (not OPENAI_API_KEY). The --json flag emits
|
||||
* JSONL events; we parse `turn.completed` for usage and `agent_message` / etc.
|
||||
* for output aggregation.
|
||||
* Captures stdout as the answer; no JSON-event parsing. Scoring is Braintrust's job.
|
||||
*/
|
||||
export class GptAdapter implements ProviderAdapter {
|
||||
readonly name = 'gpt';
|
||||
@@ -36,7 +33,7 @@ export class GptAdapter implements ProviderAdapter {
|
||||
// often run in temp dirs / non-git paths), so the read-only sandbox is now
|
||||
// the only boundary preventing codex from mutating the workdir. If you ever
|
||||
// remove `-s read-only`, drop `--skip-git-repo-check` too.
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check'];
|
||||
if (opts.model) args.push('-m', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -47,81 +44,25 @@ export class GptAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseJsonl(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
|
||||
modelUsed: opts.model ?? 'gpt',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gpt-5.4');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse codex exec --json JSONL stream.
|
||||
* Key events:
|
||||
* - item.completed with item.type === 'agent_message' → text output
|
||||
* - item.completed with item.type === 'command_execution' → tool call
|
||||
* - turn.completed → usage.input_tokens, usage.output_tokens
|
||||
* - thread.started → session id (not used here)
|
||||
*/
|
||||
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'item.completed' && obj.item) {
|
||||
if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
|
||||
output += (output ? '\n' : '') + obj.item.text;
|
||||
} else if (obj.item.type === 'command_execution') {
|
||||
toolCalls += 1;
|
||||
}
|
||||
} else if (obj.type === 'turn.completed') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_tokens ?? 0;
|
||||
out += u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines — codex stderr can leak in
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gpt-5.4',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'gpt', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
/**
|
||||
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
|
||||
*
|
||||
* Each adapter normalizes its provider's result shape into the RunResult below.
|
||||
* The benchmark runner only talks to adapters through this interface.
|
||||
* Adapters shell out to the provider's CLI and return its plain-text output. We
|
||||
* deliberately do NOT parse each vendor's proprietary JSON stream for tokens or
|
||||
* cost: that coupling is brittle (it breaks every time a vendor reshuffles its
|
||||
* output) and redundant — Braintrust owns scoring, and token/cost tracking
|
||||
* belongs to whatever instruments the actual model call. The CLI's text answer
|
||||
* is all the benchmark needs.
|
||||
*/
|
||||
|
||||
export interface RunOpts {
|
||||
@@ -18,13 +22,6 @@ export interface RunOpts {
|
||||
extraArgs?: string[];
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
input: number;
|
||||
output: number;
|
||||
/** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */
|
||||
cached?: number;
|
||||
}
|
||||
|
||||
export type RunError =
|
||||
| 'auth' // Credentials missing or invalid.
|
||||
| 'timeout' // Exceeded timeoutMs.
|
||||
@@ -33,17 +30,13 @@ export type RunError =
|
||||
| 'unknown'; // Catch-all with reason populated.
|
||||
|
||||
export interface RunResult {
|
||||
/** Provider's textual output for the prompt. */
|
||||
/** Provider's plain-text output for the prompt. */
|
||||
output: string;
|
||||
/** Normalized token usage. 0s if unreported. */
|
||||
tokens: TokenUsage;
|
||||
/** Wall-clock duration. */
|
||||
durationMs: number;
|
||||
/** Count of tool/function calls made during the run (0 if unsupported). */
|
||||
toolCalls: number;
|
||||
/** Actual model ID the provider reports using (may be a variant of the family). */
|
||||
/** Model label — the requested model or the family name. Not parsed from output. */
|
||||
modelUsed: string;
|
||||
/** If the run failed, error code + human reason. output/tokens may be partial. */
|
||||
/** If the run failed, error code + human reason. output may be empty/partial. */
|
||||
error?: { code: RunError; reason: string };
|
||||
}
|
||||
|
||||
@@ -67,6 +60,4 @@ export interface ProviderAdapter {
|
||||
available(): Promise<AvailabilityCheck>;
|
||||
/** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
|
||||
run(opts: RunOpts): Promise<RunResult>;
|
||||
/** Estimate USD cost for the reported token usage and model. */
|
||||
estimateCost(tokens: TokenUsage, model?: string): number;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user