mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 04:10:47 +02:00
Back model benchmark with Braintrust, drop in-house scoring (#13)
# Conflicts: # bun.lock # package.json
This commit is contained in:
Sinabina
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* Braintrust-backed benchmark core.
|
||||
*
|
||||
* Braintrust owns scoring, experiment tracking, comparison, and reporting. GStack
|
||||
* keeps only the unavoidable CLI-agent shims (providers/*.ts) as the eval `task`,
|
||||
* plus operational metrics (latency/tokens/cost) the CLIs report and Braintrust
|
||||
* doesn't measure for us.
|
||||
*
|
||||
* Local by default: with no BRAINTRUST_API_KEY, runs with `noSendLogs` and ships
|
||||
* nothing. Setting the key opts into the cloud dashboard (the consent boundary).
|
||||
* Runs under bun (the adapters use Bun.which), so invoke via `bun run`, never the
|
||||
* node-based `braintrust eval` CLI.
|
||||
*/
|
||||
import { Eval } from 'braintrust';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
import type { ProviderAdapter } from './providers/types';
|
||||
|
||||
export type ProviderName = 'claude' | 'gpt' | 'gemini';
|
||||
|
||||
const ADAPTERS: Record<ProviderName, () => ProviderAdapter> = {
|
||||
claude: () => new ClaudeAdapter(),
|
||||
gpt: () => new GptAdapter(),
|
||||
gemini: () => new GeminiAdapter(),
|
||||
};
|
||||
|
||||
export interface BenchCase {
|
||||
id: string;
|
||||
input: string;
|
||||
required: string[];
|
||||
}
|
||||
|
||||
export interface BenchOpts {
|
||||
timeoutMs?: number;
|
||||
/** Add an autoevals LLM judge (ClosedQA). Needs OPENAI_API_KEY. */
|
||||
judge?: boolean;
|
||||
}
|
||||
|
||||
/** Per-case operational metric Braintrust doesn't capture for a CLI subprocess: wall-clock. */
|
||||
export interface CaseOps {
|
||||
id: string;
|
||||
durationMs: number;
|
||||
modelUsed: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ProviderBenchmark {
|
||||
provider: ProviderName;
|
||||
/** Mean required-terms score 0..1 from Braintrust, or null if every case errored. */
|
||||
score: number | null;
|
||||
ops: CaseOps[];
|
||||
}
|
||||
|
||||
export const DEFAULT_CORPUS_PATH = path.join(__dirname, '..', '..', 'evals', 'model-benchmark', 'corpus.json');
|
||||
|
||||
export function loadCorpus(corpusPath = DEFAULT_CORPUS_PATH): BenchCase[] {
|
||||
return JSON.parse(fs.readFileSync(corpusPath, 'utf-8'));
|
||||
}
|
||||
|
||||
interface ScorerArgs {
|
||||
input: string;
|
||||
output: string;
|
||||
expected: string[];
|
||||
}
|
||||
type ScoreResult = { name: string; score: number };
|
||||
type Scorer = (args: ScorerArgs) => ScoreResult | Promise<ScoreResult>;
|
||||
|
||||
/** Deterministic scorer: fraction of required terms present. Replaces the old in-house evaluation.ts. */
|
||||
const requiredTerms: Scorer = ({ output, expected }) => {
|
||||
const norm = (output ?? '').toLowerCase();
|
||||
const matched = (expected ?? []).filter((t) => norm.includes(t.toLowerCase()));
|
||||
return { name: 'required-terms', score: expected?.length ? matched.length / expected.length : 1 };
|
||||
};
|
||||
|
||||
/**
|
||||
* Run one provider across the cases through Braintrust and return its score + ops.
|
||||
* The adapter is the `task`; requiredTerms (and optionally an autoevals judge) are
|
||||
* the `scores`. Empty output with zero tokens is thrown so it can't fake a 0.
|
||||
*/
|
||||
export async function runProviderBenchmark(
|
||||
provider: ProviderName,
|
||||
cases: BenchCase[],
|
||||
opts: BenchOpts = {},
|
||||
): Promise<ProviderBenchmark> {
|
||||
const factory = ADAPTERS[provider];
|
||||
if (!factory) throw new Error(`unknown provider '${provider}' (claude|gpt|gemini)`);
|
||||
const adapter = factory();
|
||||
const timeoutMs = opts.timeoutMs ?? 300_000;
|
||||
|
||||
const ops: CaseOps[] = [];
|
||||
const byInput = new Map(cases.map((c) => [c.input, c]));
|
||||
|
||||
// Braintrust calls scorers with { input, output, expected, metadata }.
|
||||
const scores: Scorer[] = [requiredTerms];
|
||||
if (opts.judge) {
|
||||
// autoevals ClosedQA = Braintrust's own LLM-judge, replacing the in-house judge.ts.
|
||||
// Normalize its result to a plain { name, score } so cross-package Score types don't clash.
|
||||
const { ClosedQA } = await import('autoevals');
|
||||
const closedQa = ClosedQA as unknown as (a: Record<string, unknown>) => Promise<{ score?: number }>;
|
||||
scores.push(async ({ input, output, expected }) => {
|
||||
const r = await closedQa({
|
||||
input,
|
||||
output,
|
||||
criteria: `Addresses the task and mentions: ${(expected ?? []).join(', ')}`,
|
||||
});
|
||||
return { name: 'judge-closedqa', score: typeof r?.score === 'number' ? r.score : 0 };
|
||||
});
|
||||
}
|
||||
|
||||
const noSendLogs = !process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
const result = await Eval(
|
||||
`gstack-model-benchmark:${provider}`,
|
||||
{
|
||||
data: () => cases.map((c) => ({ input: c.input, expected: c.required, metadata: { id: c.id } })),
|
||||
task: async (input: string) => {
|
||||
const c = byInput.get(input);
|
||||
const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-'));
|
||||
try {
|
||||
const r = await adapter.run({ prompt: input, workdir, timeoutMs });
|
||||
ops.push({
|
||||
id: c?.id ?? input.slice(0, 24),
|
||||
durationMs: r.durationMs,
|
||||
modelUsed: r.modelUsed,
|
||||
error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined,
|
||||
});
|
||||
if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`);
|
||||
// Empty output = provider never answered (silent auth/CLI failure). Fail
|
||||
// loud so it can't masquerade as a 0.0 score.
|
||||
if (!r.output.trim()) {
|
||||
throw new Error(`${provider} returned empty output (likely auth/CLI failure, not a real result)`);
|
||||
}
|
||||
return r.output;
|
||||
} finally {
|
||||
fs.rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
scores,
|
||||
},
|
||||
{ noSendLogs },
|
||||
);
|
||||
|
||||
const scoreSummary = result.summary?.scores?.['required-terms'];
|
||||
const score = typeof scoreSummary?.score === 'number' ? scoreSummary.score : null;
|
||||
return { provider, score, ops };
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
/**
|
||||
* Benchmark quality judge for multi-provider scoring.
|
||||
*
|
||||
* The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
|
||||
* the prompt + N provider outputs and scores each on: correctness, completeness,
|
||||
* code quality, edge case handling. 0-10 per dimension; overall = average.
|
||||
*
|
||||
* Judge adds ~$0.05 per benchmark run. Gated by --judge CLI flag.
|
||||
*/
|
||||
|
||||
import type { BenchmarkReport, BenchmarkEntry } from './runner';
|
||||
|
||||
export async function judgeEntries(report: BenchmarkReport): Promise<void> {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
throw new Error('ANTHROPIC_API_KEY not set — judge requires Anthropic access.');
|
||||
}
|
||||
const { default: Anthropic } = await import('@anthropic-ai/sdk').catch(() => {
|
||||
throw new Error('@anthropic-ai/sdk not installed — run `bun add @anthropic-ai/sdk` if you want the judge.');
|
||||
});
|
||||
const client = new (Anthropic as unknown as new (opts: { apiKey: string }) => {
|
||||
messages: { create: (params: Record<string, unknown>) => Promise<{ content: Array<{ type: string; text: string }> }> };
|
||||
})({ apiKey: process.env.ANTHROPIC_API_KEY! });
|
||||
|
||||
const successful = report.entries.filter(e => e.available && e.result && !e.result.error);
|
||||
if (successful.length === 0) return;
|
||||
|
||||
const judgePrompt = buildJudgePrompt(report.prompt, successful);
|
||||
const msg = await client.messages.create({
|
||||
model: 'claude-sonnet-4-6',
|
||||
max_tokens: 2048,
|
||||
messages: [{ role: 'user', content: judgePrompt }],
|
||||
});
|
||||
const textBlock = msg.content.find(c => c.type === 'text');
|
||||
if (!textBlock) return;
|
||||
|
||||
const scores = parseScores(textBlock.text, successful.length);
|
||||
for (let i = 0; i < successful.length; i++) {
|
||||
const s = scores[i];
|
||||
if (!s) continue;
|
||||
successful[i].qualityScore = s.overall;
|
||||
successful[i].qualityDetails = s.dimensions;
|
||||
}
|
||||
}
|
||||
|
||||
function buildJudgePrompt(prompt: string, entries: BenchmarkEntry[]): string {
|
||||
const lines: string[] = [
|
||||
'You are a strict, fair technical reviewer scoring N model outputs against the same prompt.',
|
||||
'',
|
||||
'--- PROMPT ---',
|
||||
prompt.length > 4000 ? prompt.slice(0, 4000) + '\n[...truncated for judge budget...]' : prompt,
|
||||
'',
|
||||
'--- OUTPUTS ---',
|
||||
];
|
||||
entries.forEach((e, i) => {
|
||||
const r = e.result!;
|
||||
const out = r.output.length > 3000 ? r.output.slice(0, 3000) + '\n[...truncated...]' : r.output;
|
||||
lines.push(`=== Output ${i + 1}: ${r.modelUsed} ===`);
|
||||
lines.push(out);
|
||||
lines.push('');
|
||||
});
|
||||
lines.push('');
|
||||
lines.push('Score each output on these dimensions (0-10 per dimension):');
|
||||
lines.push(' - correctness: does it solve what the prompt asked?');
|
||||
lines.push(' - completeness: are edge cases and error paths addressed?');
|
||||
lines.push(' - code_quality: naming, structure, explicitness');
|
||||
lines.push(' - edge_cases: handling of nil/empty/invalid input');
|
||||
lines.push('');
|
||||
lines.push('Return JSON only, in this exact shape:');
|
||||
lines.push('{"scores":[');
|
||||
lines.push(' {"output":1,"correctness":N,"completeness":N,"code_quality":N,"edge_cases":N,"overall":N,"notes":"..."},');
|
||||
lines.push(' ...');
|
||||
lines.push(']}');
|
||||
lines.push('');
|
||||
lines.push('overall = rounded average of the 4 dimensions. No other commentary.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
interface ParsedScore {
|
||||
overall: number;
|
||||
dimensions: Record<string, number>;
|
||||
}
|
||||
|
||||
function parseScores(raw: string, expectedCount: number): ParsedScore[] {
|
||||
const match = raw.match(/\{[\s\S]*\}/);
|
||||
if (!match) return [];
|
||||
try {
|
||||
const obj = JSON.parse(match[0]);
|
||||
if (!Array.isArray(obj.scores)) return [];
|
||||
return obj.scores.slice(0, expectedCount).map((s: Record<string, number>) => ({
|
||||
overall: Number(s.overall ?? 0),
|
||||
dimensions: {
|
||||
correctness: Number(s.correctness ?? 0),
|
||||
completeness: Number(s.completeness ?? 0),
|
||||
code_quality: Number(s.code_quality ?? 0),
|
||||
edge_cases: Number(s.edge_cases ?? 0),
|
||||
},
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
/**
|
||||
* Per-model pricing tables.
|
||||
*
|
||||
* Prices are USD per million tokens as of `as_of`. Update quarterly.
|
||||
* Link to provider pricing pages:
|
||||
* - Anthropic: https://www.anthropic.com/pricing#api
|
||||
* - OpenAI: https://openai.com/api/pricing/
|
||||
* - Google AI: https://ai.google.dev/pricing
|
||||
*
|
||||
* When a model isn't in the table, estimateCost returns 0 with a console warning.
|
||||
* Prefer adding a new row to the table over guessing.
|
||||
*/
|
||||
|
||||
export interface ModelPricing {
|
||||
input_per_mtok: number;
|
||||
output_per_mtok: number;
|
||||
as_of: string; // YYYY-MM
|
||||
}
|
||||
|
||||
export const PRICING: Record<string, ModelPricing> = {
|
||||
// Claude (Anthropic)
|
||||
'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' },
|
||||
'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' },
|
||||
'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
|
||||
// OpenAI (GPT + o-series)
|
||||
'gpt-5.4': { input_per_mtok: 2.50, output_per_mtok: 10.00, as_of: '2026-04' },
|
||||
'gpt-5.4-mini': { input_per_mtok: 0.60, output_per_mtok: 2.40, as_of: '2026-04' },
|
||||
'o3': { input_per_mtok: 15.00, output_per_mtok: 60.00, as_of: '2026-04' },
|
||||
'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' },
|
||||
|
||||
// Google
|
||||
'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' },
|
||||
};
|
||||
|
||||
const WARNED = new Set<string>();
|
||||
|
||||
export function estimateCostUsd(
|
||||
tokens: { input: number; output: number; cached?: number },
|
||||
model: string | undefined
|
||||
): number {
|
||||
if (!model) return 0;
|
||||
const row = PRICING[model];
|
||||
if (!row) {
|
||||
if (!WARNED.has(model)) {
|
||||
WARNED.add(model);
|
||||
console.error(`WARN: no pricing for model ${model}; returning 0. Add it to lib/model-benchmark/pricing.ts.`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
// Anthropic and OpenAI report cached tokens as a separate (disjoint) field from
|
||||
// uncached input tokens. tokens.input is already the uncached portion; tokens.cached
|
||||
// is the cache-read count billed at 10% of the regular input rate. Do NOT subtract
|
||||
// cached from input — they don't overlap.
|
||||
const cachedDiscount = 0.1;
|
||||
const inputCost = tokens.input * row.input_per_mtok / 1_000_000;
|
||||
const cachedCost = (tokens.cached ?? 0) * row.input_per_mtok * cachedDiscount / 1_000_000;
|
||||
const outputCost = tokens.output * row.output_per_mtok / 1_000_000;
|
||||
return +(inputCost + cachedCost + outputCost).toFixed(6);
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -7,12 +6,10 @@ import * as os from 'os';
|
||||
import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
|
||||
|
||||
/**
|
||||
* Claude adapter — wraps the `claude` CLI via claude -p.
|
||||
* Claude adapter — wraps the `claude` CLI via `claude -p` in plain-text mode.
|
||||
*
|
||||
* For brevity and to avoid duplicating the full stream-json parser, this adapter
|
||||
* uses claude CLI in non-interactive mode (--print) with the simpler JSON output
|
||||
* format. If richer event-level metrics are needed (per-tool timing etc.),
|
||||
* swap to session-runner's full stream-json parser.
|
||||
* We capture stdout as the answer and do not parse Claude's JSON output shape:
|
||||
* scoring is Braintrust's job and it only needs the text.
|
||||
*/
|
||||
export class ClaudeAdapter implements ProviderAdapter {
|
||||
readonly name = 'claude';
|
||||
@@ -41,7 +38,7 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
if (!resolved) {
|
||||
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
|
||||
}
|
||||
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
|
||||
const args = [...resolved.argsPrefix, '-p'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -56,70 +53,25 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
||||
});
|
||||
const parsed = this.parseOutput(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
|
||||
modelUsed: opts.model ?? 'claude',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse claude -p --output-format json output. Shape (as of 2026-04):
|
||||
* { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
|
||||
* num_turns, session_id, ... }
|
||||
* Older formats may differ — adapter is best-effort.
|
||||
*/
|
||||
private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
|
||||
try {
|
||||
const obj = JSON.parse(raw);
|
||||
const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
|
||||
const u = obj.usage ?? {};
|
||||
return {
|
||||
output: result,
|
||||
tokens: {
|
||||
input: u.input_tokens ?? 0,
|
||||
output: u.output_tokens ?? 0,
|
||||
cached: u.cache_read_input_tokens,
|
||||
},
|
||||
toolCalls: obj.num_turns ?? 0,
|
||||
modelUsed: obj.model,
|
||||
};
|
||||
} catch {
|
||||
// Non-JSON output: treat as plain text.
|
||||
return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'claude-opus-4-7',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'claude', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
* Gemini adapter — wraps the `gemini` CLI in plain-text mode.
|
||||
*
|
||||
* Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
|
||||
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
|
||||
* stream-json` is requested. This adapter uses a single-response form for simplicity
|
||||
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
|
||||
* Auth comes from its OAuth files, ~/.gemini/.env, or GOOGLE_API_KEY/GEMINI_API_KEY.
|
||||
* We capture stdout as the answer and do not parse Gemini's stream-json schema —
|
||||
* that coupling broke every time Gemini reshuffled its event shape.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
readonly name = 'gemini';
|
||||
@@ -25,19 +23,23 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
|
||||
const newCfgDir = path.join(os.homedir(), '.gemini');
|
||||
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
|
||||
const geminiEnv = path.join(newCfgDir, '.env');
|
||||
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
|
||||
const hasKey = !!process.env.GOOGLE_API_KEY;
|
||||
const hasEnvFileKey = fs.existsSync(geminiEnv)
|
||||
&& /^(?:GOOGLE_API_KEY|GEMINI_API_KEY)\s*=/m.test(fs.readFileSync(geminiEnv, 'utf-8'));
|
||||
const hasKey = !!process.env.GOOGLE_API_KEY || !!process.env.GEMINI_API_KEY || hasEnvFileKey;
|
||||
if (!hasCfg && !hasKey) {
|
||||
return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
|
||||
return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY/GEMINI_API_KEY.' };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
// Default to --yolo (non-interactive) and stream-json output so we can parse
|
||||
// tokens + tool calls. Callers can override via extraArgs.
|
||||
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
|
||||
// --skip-trust lets the CLI run in disposable/non-git benchmark workdirs.
|
||||
// Plain -p is non-interactive; benchmark prompts are reasoning tasks, not file
|
||||
// ops, and the workdir is a throwaway temp dir.
|
||||
const args = ['-p', opts.prompt, '--skip-trust'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -48,78 +50,25 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseStreamJson(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
|
||||
modelUsed: opts.model ?? 'gemini',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login|api key/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse gemini NDJSON stream events:
|
||||
* init → session id (discarded here)
|
||||
* message { delta: true, text } → concat to output
|
||||
* tool_use { name } → increment toolCalls
|
||||
* result { usage: { input_token_count, output_token_count } } → tokens
|
||||
*/
|
||||
private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'message' && typeof obj.text === 'string') {
|
||||
output += obj.text;
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_token_count ?? u.prompt_tokens ?? 0;
|
||||
out += u.output_token_count ?? u.completion_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gemini-2.5-pro',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login|api key/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429|quota/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'gemini', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (`codex exec`) in plain-text mode.
|
||||
*
|
||||
* Codex uses ~/.codex/ for auth (not OPENAI_API_KEY). The --json flag emits
|
||||
* JSONL events; we parse `turn.completed` for usage and `agent_message` / etc.
|
||||
* for output aggregation.
|
||||
* Captures stdout as the answer; no JSON-event parsing. Scoring is Braintrust's job.
|
||||
*/
|
||||
export class GptAdapter implements ProviderAdapter {
|
||||
readonly name = 'gpt';
|
||||
@@ -36,7 +33,7 @@ export class GptAdapter implements ProviderAdapter {
|
||||
// often run in temp dirs / non-git paths), so the read-only sandbox is now
|
||||
// the only boundary preventing codex from mutating the workdir. If you ever
|
||||
// remove `-s read-only`, drop `--skip-git-repo-check` too.
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check'];
|
||||
if (opts.model) args.push('-m', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -47,81 +44,25 @@ export class GptAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseJsonl(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
|
||||
modelUsed: opts.model ?? 'gpt',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gpt-5.4');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse codex exec --json JSONL stream.
|
||||
* Key events:
|
||||
* - item.completed with item.type === 'agent_message' → text output
|
||||
* - item.completed with item.type === 'command_execution' → tool call
|
||||
* - turn.completed → usage.input_tokens, usage.output_tokens
|
||||
* - thread.started → session id (not used here)
|
||||
*/
|
||||
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'item.completed' && obj.item) {
|
||||
if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
|
||||
output += (output ? '\n' : '') + obj.item.text;
|
||||
} else if (obj.item.type === 'command_execution') {
|
||||
toolCalls += 1;
|
||||
}
|
||||
} else if (obj.type === 'turn.completed') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_tokens ?? 0;
|
||||
out += u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines — codex stderr can leak in
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gpt-5.4',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'gpt', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
/**
|
||||
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
|
||||
*
|
||||
* Each adapter normalizes its provider's result shape into the RunResult below.
|
||||
* The benchmark runner only talks to adapters through this interface.
|
||||
* Adapters shell out to the provider's CLI and return its plain-text output. We
|
||||
* deliberately do NOT parse each vendor's proprietary JSON stream for tokens or
|
||||
* cost: that coupling is brittle (it breaks every time a vendor reshuffles its
|
||||
* output) and redundant — Braintrust owns scoring, and token/cost tracking
|
||||
* belongs to whatever instruments the actual model call. The CLI's text answer
|
||||
* is all the benchmark needs.
|
||||
*/
|
||||
|
||||
export interface RunOpts {
|
||||
@@ -18,13 +22,6 @@ export interface RunOpts {
|
||||
extraArgs?: string[];
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
input: number;
|
||||
output: number;
|
||||
/** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */
|
||||
cached?: number;
|
||||
}
|
||||
|
||||
export type RunError =
|
||||
| 'auth' // Credentials missing or invalid.
|
||||
| 'timeout' // Exceeded timeoutMs.
|
||||
@@ -33,17 +30,13 @@ export type RunError =
|
||||
| 'unknown'; // Catch-all with reason populated.
|
||||
|
||||
export interface RunResult {
|
||||
/** Provider's textual output for the prompt. */
|
||||
/** Provider's plain-text output for the prompt. */
|
||||
output: string;
|
||||
/** Normalized token usage. 0s if unreported. */
|
||||
tokens: TokenUsage;
|
||||
/** Wall-clock duration. */
|
||||
durationMs: number;
|
||||
/** Count of tool/function calls made during the run (0 if unsupported). */
|
||||
toolCalls: number;
|
||||
/** Actual model ID the provider reports using (may be a variant of the family). */
|
||||
/** Model label — the requested model or the family name. Not parsed from output. */
|
||||
modelUsed: string;
|
||||
/** If the run failed, error code + human reason. output/tokens may be partial. */
|
||||
/** If the run failed, error code + human reason. output may be empty/partial. */
|
||||
error?: { code: RunError; reason: string };
|
||||
}
|
||||
|
||||
@@ -67,6 +60,4 @@ export interface ProviderAdapter {
|
||||
available(): Promise<AvailabilityCheck>;
|
||||
/** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
|
||||
run(opts: RunOpts): Promise<RunResult>;
|
||||
/** Estimate USD cost for the reported token usage and model. */
|
||||
estimateCost(tokens: TokenUsage, model?: string): number;
|
||||
}
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
/**
|
||||
* Multi-provider benchmark runner.
|
||||
*
|
||||
* Orchestrates running the same prompt across multiple provider adapters and
|
||||
* aggregates RunResult outputs + judge scores into a single report. Adapters
|
||||
* run in parallel (Promise.allSettled) so a slow provider doesn't block a fast
|
||||
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
|
||||
*/
|
||||
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
|
||||
export interface BenchmarkInput {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
timeoutMs?: number;
|
||||
/** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */
|
||||
providers: Array<'claude' | 'gpt' | 'gemini'>;
|
||||
/** Optional per-provider model overrides. */
|
||||
models?: Partial<Record<'claude' | 'gpt' | 'gemini', string>>;
|
||||
/** If true, skip providers whose available() returns !ok. If false, include them with error. */
|
||||
skipUnavailable?: boolean;
|
||||
}
|
||||
|
||||
export interface BenchmarkEntry {
|
||||
provider: string;
|
||||
family: 'claude' | 'gpt' | 'gemini';
|
||||
available: boolean;
|
||||
unavailable_reason?: string;
|
||||
result?: RunResult;
|
||||
costUsd?: number;
|
||||
/** Judge score 0-10 across dimensions. Populated separately by the judge step. */
|
||||
qualityScore?: number;
|
||||
qualityDetails?: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface BenchmarkReport {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
startedAt: string;
|
||||
durationMs: number;
|
||||
entries: BenchmarkEntry[];
|
||||
}
|
||||
|
||||
const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = {
|
||||
claude: () => new ClaudeAdapter(),
|
||||
gpt: () => new GptAdapter(),
|
||||
gemini: () => new GeminiAdapter(),
|
||||
};
|
||||
|
||||
export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkReport> {
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const timeoutMs = input.timeoutMs ?? 300_000;
|
||||
|
||||
const entries: BenchmarkEntry[] = [];
|
||||
const runPromises: Array<Promise<void>> = [];
|
||||
|
||||
for (const name of input.providers) {
|
||||
const factory = ADAPTERS[name];
|
||||
if (!factory) {
|
||||
entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` });
|
||||
continue;
|
||||
}
|
||||
const adapter = factory();
|
||||
const entry: BenchmarkEntry = { provider: adapter.name, family: adapter.family, available: true };
|
||||
entries.push(entry);
|
||||
|
||||
runPromises.push((async () => {
|
||||
const check = await adapter.available();
|
||||
entry.available = check.ok;
|
||||
if (!check.ok) {
|
||||
entry.unavailable_reason = check.reason;
|
||||
if (input.skipUnavailable) return;
|
||||
}
|
||||
const opts: RunOpts = {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
timeoutMs,
|
||||
model: input.models?.[name],
|
||||
};
|
||||
const res = await adapter.run(opts);
|
||||
entry.result = res;
|
||||
entry.costUsd = adapter.estimateCost(res.tokens, res.modelUsed);
|
||||
})());
|
||||
}
|
||||
|
||||
await Promise.allSettled(runPromises);
|
||||
|
||||
return {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
startedAt,
|
||||
durationMs: Date.now() - startedAtMs,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
|
||||
export function formatJson(report: BenchmarkReport): string {
|
||||
return JSON.stringify(report, null, 2);
|
||||
}
|
||||
|
||||
export function formatMarkdown(report: BenchmarkReport): string {
|
||||
const lines: string[] = [
|
||||
`# Benchmark report — ${report.startedAt}`,
|
||||
'',
|
||||
`**Prompt:** ${report.prompt.length > 200 ? report.prompt.slice(0, 200) + '…' : report.prompt}`,
|
||||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
lines.push(`| ${e.provider} | - | - | - | - | - | unavailable: ${e.unavailable_reason ?? 'unknown'} |`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function msToStr(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function fmtCost(usd?: number): string {
|
||||
if (usd === undefined) return '-';
|
||||
if (usd < 0.01) return `$${usd.toFixed(4)}`;
|
||||
return `$${usd.toFixed(2)}`;
|
||||
}
|
||||
Reference in New Issue
Block a user