diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index 1a86d92e8..e4c906fbd 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -1,33 +1,37 @@ #!/usr/bin/env bun /** - * gstack-model-benchmark — run the same prompt across multiple providers - * and compare latency, tokens, cost, quality, and tool-call count. + * gstack-model-benchmark — run the same prompt(s) across providers and compare, + * with scoring, experiments, and reporting owned by Braintrust (local by default; + * nothing is uploaded unless BRAINTRUST_API_KEY is set — that key is the consent + * boundary for the cloud dashboard). * * Usage: - * gstack-model-benchmark [options] + * gstack-model-benchmark [] [options] * * Options: * --models claude,gpt,gemini Comma-separated provider list (default: claude) - * --prompt "" Inline prompt instead of a file - * --workdir Working dir passed to each CLI (default: cwd) - * --timeout-ms Per-provider timeout (default: 300000) - * --output table|json|markdown Output format (default: table) - * --skip-unavailable Skip providers that fail available() check - * (default: include them with unavailable marker) - * --judge Run Anthropic SDK judge on outputs for quality score - * (requires ANTHROPIC_API_KEY; adds ~$0.05 per call) + * --prompt "" Ad-hoc single-case prompt instead of the corpus + * --corpus Corpus JSON (default: evals/model-benchmark/corpus.json) + * --timeout-ms Per-provider-per-case timeout (default: 300000) + * --output table|json Output format (default: table) + * --judge Add the autoevals ClosedQA LLM judge (needs OPENAI_API_KEY) * --dry-run Validate flags + resolve auth, don't invoke providers * * Examples: * gstack-model-benchmark --prompt "Write a haiku about databases" --models claude,gpt - * gstack-model-benchmark ./test-prompt.txt --models claude,gpt,gemini --judge + * gstack-model-benchmark --models claude,gpt,gemini # runs the corpus * gstack-model-benchmark --prompt "hi" --models claude,gpt,gemini --dry-run */ import '../lib/conductor-env-shim'; import * as fs from 'fs'; -import * as path from 'path'; -import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkInput } from '../lib/model-benchmark/runner'; +import { + loadCorpus, + runProviderBenchmark, + type BenchCase, + type ProviderBenchmark, + type ProviderName, +} from '../lib/model-benchmark/braintrust-eval'; import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude'; import { GptAdapter } from '../lib/model-benchmark/providers/gpt'; import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini'; @@ -38,10 +42,10 @@ const ADAPTER_FACTORIES = { gemini: () => new GeminiAdapter(), }; -type OutputFormat = 'table' | 'json' | 'markdown'; +type OutputFormat = 'table' | 'json'; const CLI_ARGS = process.argv.slice(2); -const VALUE_FLAGS = new Set(['--models', '--prompt', '--workdir', '--timeout-ms', '--output']); +const VALUE_FLAGS = new Set(['--models', '--prompt', '--corpus', '--timeout-ms', '--output']); function arg(name: string, def?: string): string | undefined { const idx = CLI_ARGS.findIndex(a => a === name || a.startsWith(name + '=')); @@ -76,93 +80,92 @@ function positionalArgs(args: string[]): string[] { return positional; } -function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> { +function parseProviders(s: string | undefined): ProviderName[] { if (!s) return ['claude']; - const seen = new Set<'claude' | 'gpt' | 'gemini'>(); + const seen = new Set(); for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) { if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p); - else { - console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); - } + else console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); } return seen.size ? Array.from(seen) : ['claude']; } -function resolvePrompt(positional: string | undefined): string { +/** Resolve the cases: --prompt/file → single ad-hoc case; else the corpus. */ +function resolveCases(positional: string | undefined): BenchCase[] { const inline = arg('--prompt'); - if (inline) return inline; - if (!positional) { - console.error('ERROR: specify a prompt via positional path or --prompt ""'); - process.exit(1); + if (inline) return [{ id: 'adhoc', input: inline, required: [] }]; + if (positional && fs.existsSync(positional)) { + return [{ id: 'adhoc', input: fs.readFileSync(positional, 'utf-8'), required: [] }]; } - if (fs.existsSync(positional)) { - return fs.readFileSync(positional, 'utf-8'); - } - // Not a file — treat as inline prompt - return positional; + if (positional) return [{ id: 'adhoc', input: positional, required: [] }]; + return loadCorpus(arg('--corpus')); } async function main(): Promise { const positional = positionalArgs(CLI_ARGS)[0]; - const prompt = resolvePrompt(positional); const providers = parseProviders(arg('--models')); - const workdir = arg('--workdir', process.cwd())!; const timeoutMs = parseInt(arg('--timeout-ms', '300000')!, 10); const output = (arg('--output', 'table') as OutputFormat); - const skipUnavailable = flag('--skip-unavailable'); const doJudge = flag('--judge'); const dryRun = flag('--dry-run'); if (dryRun) { - await dryRunReport({ prompt, providers, workdir, timeoutMs, output, doJudge }); + await dryRunReport({ cases: resolveCases(positional), providers, timeoutMs, output, doJudge }); return; } - const input: BenchmarkInput = { - prompt, - workdir, - providers, - timeoutMs, - skipUnavailable, - }; - - const report = await runBenchmark(input); - - if (doJudge) { - try { - const { judgeEntries } = await import('../lib/model-benchmark/judge'); - await judgeEntries(report); - } catch (err) { - console.error(`WARN: judge unavailable: ${(err as Error).message}`); - } + if (doJudge && !process.env.OPENAI_API_KEY) { + console.error('WARN: --judge needs OPENAI_API_KEY for the autoevals ClosedQA judge — running with the deterministic scorer only.'); } - let out: string; - switch (output) { - case 'json': out = formatJson(report); break; - case 'markdown': out = formatMarkdown(report); break; - case 'table': - default: out = formatTable(report); break; + const cases = resolveCases(positional); + const results: ProviderBenchmark[] = []; + for (const provider of providers) { + results.push(await runProviderBenchmark(provider, cases, { timeoutMs, judge: doJudge && !!process.env.OPENAI_API_KEY })); } - process.stdout.write(out + '\n'); + + process.stdout.write((output === 'json' ? JSON.stringify(results, null, 2) : formatTable(results)) + '\n'); +} + +function formatTable(results: ProviderBenchmark[]): string { + const header = `Provider Score Cases Latency(avg) Tokens(in→out) Cost Errors`; + const rows: string[] = [header, '-'.repeat(header.length)]; + for (const r of results) { + const n = r.ops.length || 1; + const avgMs = r.ops.reduce((s, o) => s + o.durationMs, 0) / n; + const tin = r.ops.reduce((s, o) => s + o.tokensIn, 0); + const tout = r.ops.reduce((s, o) => s + o.tokensOut, 0); + const cost = r.ops.reduce((s, o) => s + o.costUsd, 0); + const errs = r.ops.filter(o => o.error).length; + const score = r.score === null ? '—' : `${(r.score * 100).toFixed(1)}%`; + rows.push( + `${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${pad(`${tin}→${tout}`, 15)} ${pad(fmtCost(cost), 9)} ${errs || ''}`, + ); + } + return rows.join('\n'); } async function dryRunReport(opts: { - prompt: string; - providers: Array<'claude' | 'gpt' | 'gemini'>; - workdir: string; + cases: BenchCase[]; + providers: ProviderName[]; timeoutMs: number; output: OutputFormat; doJudge: boolean; }): Promise { + const adhoc = opts.cases.length === 1 && opts.cases[0].id === 'adhoc'; const lines: string[] = []; lines.push('== gstack-model-benchmark --dry-run =='); - lines.push(` prompt: ${opts.prompt.length > 80 ? opts.prompt.slice(0, 80) + '…' : opts.prompt}`); + if (adhoc) { + const p = opts.cases[0].input; + lines.push(` prompt: ${p.length > 80 ? p.slice(0, 80) + '…' : p}`); + } else { + lines.push(` corpus: ${opts.cases.length} case(s)`); + } lines.push(` providers: ${opts.providers.join(', ')}`); - lines.push(` workdir: ${opts.workdir}`); lines.push(` timeout_ms: ${opts.timeoutMs}`); lines.push(` output: ${opts.output}`); - lines.push(` judge: ${opts.doJudge ? 'on (Anthropic SDK)' : 'off'}`); + lines.push(` judge: ${opts.doJudge ? 'on (autoevals ClosedQA)' : 'off'}`); + lines.push(` upload: ${process.env.BRAINTRUST_API_KEY ? 'ON — BRAINTRUST_API_KEY set (cloud dashboard)' : 'off (local only, nothing uploaded)'}`); lines.push(''); lines.push('Adapter availability:'); let authFailures = 0; @@ -173,20 +176,28 @@ async function dryRunReport(opts: { authFailures += 1; continue; } - const adapter = factory(); - const check = await adapter.available(); - if (check.ok) { - lines.push(` ${adapter.name}: OK`); - } else { - lines.push(` ${adapter.name}: NOT READY — ${check.reason}`); - authFailures += 1; - } + const check = await factory().available(); + if (check.ok) lines.push(` ${name}: OK`); + else { lines.push(` ${name}: NOT READY — ${check.reason}`); authFailures += 1; } } lines.push(''); lines.push(`(--dry-run — no prompts sent. ${authFailures} provider(s) unavailable.)`); process.stdout.write(lines.join('\n') + '\n'); } +function pad(s: string, n: number): string { + return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length); +} +function msToStr(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + return `${(ms / 1000).toFixed(1)}s`; +} +function fmtCost(usd: number): string { + if (usd === 0) return '$0'; + if (usd < 0.01) return `$${usd.toFixed(4)}`; + return `$${usd.toFixed(2)}`; +} + main().catch(err => { console.error('FATAL:', err); process.exit(1); diff --git a/evals/model-benchmark/corpus.json b/evals/model-benchmark/corpus.json new file mode 100644 index 000000000..4ec490311 --- /dev/null +++ b/evals/model-benchmark/corpus.json @@ -0,0 +1,17 @@ +[ + { + "id": "security-review", + "input": "Review a synthetic patch containing command injection, unsafe recursive delete, and path traversal. Do not mutate files. Report only.", + "required": ["command injection", "recursive delete", "path traversal", "no mutation"] + }, + { + "id": "graph-indexer-consent", + "input": "Decide whether GStack should require a third-party graph indexer. Justify the install policy.", + "required": ["optional", "consent", "fallback"] + }, + { + "id": "resolver-safety", + "input": "Diagnose a resolver that passes an untrusted logical skill name into path.resolve. Recommend the fix.", + "required": ["allowlist", "single segment", "traversal"] + } +] diff --git a/evals/model-benchmark/gstack.eval.ts b/evals/model-benchmark/gstack.eval.ts new file mode 100644 index 000000000..b93f59ff5 --- /dev/null +++ b/evals/model-benchmark/gstack.eval.ts @@ -0,0 +1,21 @@ +/** + * GStack model benchmark — Braintrust entrypoint for `bun run` / CI. + * + * Braintrust owns scoring, comparison, and reporting. See + * lib/model-benchmark/braintrust-eval.ts for the core; the user-facing CLI is + * bin/gstack-model-benchmark. + * + * Local run, nothing leaves the machine: + * GSTACK_BENCH_PROVIDER=claude bun run evals/model-benchmark/gstack.eval.ts + * + * Opt into the Braintrust dashboard (consent-gated cloud): + * export BRAINTRUST_API_KEY=... + * for p in claude gpt gemini; do GSTACK_BENCH_PROVIDER=$p bun run evals/model-benchmark/gstack.eval.ts; done + */ +import { loadCorpus, runProviderBenchmark, type ProviderName } from '../../lib/model-benchmark/braintrust-eval'; + +const provider = (process.env.GSTACK_BENCH_PROVIDER ?? 'claude') as ProviderName; +const timeoutMs = Number(process.env.GSTACK_BENCH_TIMEOUT_MS ?? 300_000); +const judge = process.env.GSTACK_BENCH_JUDGE === '1'; + +await runProviderBenchmark(provider, loadCorpus(), { timeoutMs, judge }); diff --git a/lib/model-benchmark/braintrust-eval.ts b/lib/model-benchmark/braintrust-eval.ts new file mode 100644 index 000000000..af58558ce --- /dev/null +++ b/lib/model-benchmark/braintrust-eval.ts @@ -0,0 +1,160 @@ +/** + * Braintrust-backed benchmark core. + * + * Braintrust owns scoring, experiment tracking, comparison, and reporting. GStack + * keeps only the unavoidable CLI-agent shims (providers/*.ts) as the eval `task`, + * plus operational metrics (latency/tokens/cost) the CLIs report and Braintrust + * doesn't measure for us. + * + * Local by default: with no BRAINTRUST_API_KEY, runs with `noSendLogs` and ships + * nothing. Setting the key opts into the cloud dashboard (the consent boundary). + * Runs under bun (the adapters use Bun.which), so invoke via `bun run`, never the + * node-based `braintrust eval` CLI. + */ +import { Eval } from 'braintrust'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ClaudeAdapter } from './providers/claude'; +import { GptAdapter } from './providers/gpt'; +import { GeminiAdapter } from './providers/gemini'; +import type { ProviderAdapter } from './providers/types'; + +export type ProviderName = 'claude' | 'gpt' | 'gemini'; + +const ADAPTERS: Record ProviderAdapter> = { + claude: () => new ClaudeAdapter(), + gpt: () => new GptAdapter(), + gemini: () => new GeminiAdapter(), +}; + +export interface BenchCase { + id: string; + input: string; + required: string[]; +} + +export interface BenchOpts { + timeoutMs?: number; + /** Add an autoevals LLM judge (ClosedQA). Needs OPENAI_API_KEY. */ + judge?: boolean; +} + +/** Per-case operational metrics the adapter reports — Braintrust doesn't time the CLI for us. */ +export interface CaseOps { + id: string; + durationMs: number; + tokensIn: number; + tokensOut: number; + costUsd: number; + toolCalls: number; + modelUsed: string; + error?: string; +} + +export interface ProviderBenchmark { + provider: ProviderName; + /** Mean required-terms score 0..1 from Braintrust, or null if every case errored. */ + score: number | null; + ops: CaseOps[]; +} + +export const DEFAULT_CORPUS_PATH = path.join(__dirname, '..', '..', 'evals', 'model-benchmark', 'corpus.json'); + +export function loadCorpus(corpusPath = DEFAULT_CORPUS_PATH): BenchCase[] { + return JSON.parse(fs.readFileSync(corpusPath, 'utf-8')); +} + +interface ScorerArgs { + input: string; + output: string; + expected: string[]; +} +type ScoreResult = { name: string; score: number }; +type Scorer = (args: ScorerArgs) => ScoreResult | Promise; + +/** Deterministic scorer: fraction of required terms present. Replaces the old in-house evaluation.ts. */ +const requiredTerms: Scorer = ({ output, expected }) => { + const norm = (output ?? '').toLowerCase(); + const matched = (expected ?? []).filter((t) => norm.includes(t.toLowerCase())); + return { name: 'required-terms', score: expected?.length ? matched.length / expected.length : 1 }; +}; + +/** + * Run one provider across the cases through Braintrust and return its score + ops. + * The adapter is the `task`; requiredTerms (and optionally an autoevals judge) are + * the `scores`. Empty output with zero tokens is thrown so it can't fake a 0. + */ +export async function runProviderBenchmark( + provider: ProviderName, + cases: BenchCase[], + opts: BenchOpts = {}, +): Promise { + const factory = ADAPTERS[provider]; + if (!factory) throw new Error(`unknown provider '${provider}' (claude|gpt|gemini)`); + const adapter = factory(); + const timeoutMs = opts.timeoutMs ?? 300_000; + + const ops: CaseOps[] = []; + const byInput = new Map(cases.map((c) => [c.input, c])); + + // Braintrust calls scorers with { input, output, expected, metadata }. + const scores: Scorer[] = [requiredTerms]; + if (opts.judge) { + // autoevals ClosedQA = Braintrust's own LLM-judge, replacing the in-house judge.ts. + // Normalize its result to a plain { name, score } so cross-package Score types don't clash. + const { ClosedQA } = await import('autoevals'); + const closedQa = ClosedQA as unknown as (a: Record) => Promise<{ score?: number }>; + scores.push(async ({ input, output, expected }) => { + const r = await closedQa({ + input, + output, + criteria: `Addresses the task and mentions: ${(expected ?? []).join(', ')}`, + }); + return { name: 'judge-closedqa', score: typeof r?.score === 'number' ? r.score : 0 }; + }); + } + + const noSendLogs = !process.env.BRAINTRUST_API_KEY; + + const result = await Eval( + `gstack-model-benchmark:${provider}`, + { + data: () => cases.map((c) => ({ input: c.input, expected: c.required, metadata: { id: c.id } })), + task: async (input: string) => { + const c = byInput.get(input); + const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-')); + try { + const r = await adapter.run({ prompt: input, workdir, timeoutMs }); + const costUsd = adapter.estimateCost(r.tokens, r.modelUsed); + const op: CaseOps = { + id: c?.id ?? input.slice(0, 24), + durationMs: r.durationMs, + tokensIn: r.tokens.input, + tokensOut: r.tokens.output, + costUsd, + toolCalls: r.toolCalls, + modelUsed: r.modelUsed, + error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined, + }; + ops.push(op); + if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`); + // Empty output + zero tokens = provider never answered (silent auth/CLI failure). + // Fail loud so it can't masquerade as a 0.0 score. + if (!r.output.trim() && r.tokens.input === 0 && r.tokens.output === 0) { + throw new Error(`${provider} returned empty output with zero tokens (likely auth/CLI failure, not a real result)`); + } + return r.output; + } finally { + fs.rmSync(workdir, { recursive: true, force: true }); + } + }, + scores, + }, + { noSendLogs }, + ); + + const scoreSummary = result.summary?.scores?.['required-terms']; + const score = typeof scoreSummary?.score === 'number' ? scoreSummary.score : null; + return { provider, score, ops }; +} diff --git a/lib/model-benchmark/judge.ts b/lib/model-benchmark/judge.ts deleted file mode 100644 index 525b8c0ad..000000000 --- a/lib/model-benchmark/judge.ts +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Benchmark quality judge for multi-provider scoring. - * - * The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees - * the prompt + N provider outputs and scores each on: correctness, completeness, - * code quality, edge case handling. 0-10 per dimension; overall = average. - * - * Judge adds ~$0.05 per benchmark run. Gated by --judge CLI flag. - */ - -import type { BenchmarkReport, BenchmarkEntry } from './runner'; - -export async function judgeEntries(report: BenchmarkReport): Promise { - if (!process.env.ANTHROPIC_API_KEY) { - throw new Error('ANTHROPIC_API_KEY not set — judge requires Anthropic access.'); - } - const { default: Anthropic } = await import('@anthropic-ai/sdk').catch(() => { - throw new Error('@anthropic-ai/sdk not installed — run `bun add @anthropic-ai/sdk` if you want the judge.'); - }); - const client = new (Anthropic as unknown as new (opts: { apiKey: string }) => { - messages: { create: (params: Record) => Promise<{ content: Array<{ type: string; text: string }> }> }; - })({ apiKey: process.env.ANTHROPIC_API_KEY! }); - - const successful = report.entries.filter(e => e.available && e.result && !e.result.error); - if (successful.length === 0) return; - - const judgePrompt = buildJudgePrompt(report.prompt, successful); - const msg = await client.messages.create({ - model: 'claude-sonnet-4-6', - max_tokens: 2048, - messages: [{ role: 'user', content: judgePrompt }], - }); - const textBlock = msg.content.find(c => c.type === 'text'); - if (!textBlock) return; - - const scores = parseScores(textBlock.text, successful.length); - for (let i = 0; i < successful.length; i++) { - const s = scores[i]; - if (!s) continue; - successful[i].qualityScore = s.overall; - successful[i].qualityDetails = s.dimensions; - } -} - -function buildJudgePrompt(prompt: string, entries: BenchmarkEntry[]): string { - const lines: string[] = [ - 'You are a strict, fair technical reviewer scoring N model outputs against the same prompt.', - '', - '--- PROMPT ---', - prompt.length > 4000 ? prompt.slice(0, 4000) + '\n[...truncated for judge budget...]' : prompt, - '', - '--- OUTPUTS ---', - ]; - entries.forEach((e, i) => { - const r = e.result!; - const out = r.output.length > 3000 ? r.output.slice(0, 3000) + '\n[...truncated...]' : r.output; - lines.push(`=== Output ${i + 1}: ${r.modelUsed} ===`); - lines.push(out); - lines.push(''); - }); - lines.push(''); - lines.push('Score each output on these dimensions (0-10 per dimension):'); - lines.push(' - correctness: does it solve what the prompt asked?'); - lines.push(' - completeness: are edge cases and error paths addressed?'); - lines.push(' - code_quality: naming, structure, explicitness'); - lines.push(' - edge_cases: handling of nil/empty/invalid input'); - lines.push(''); - lines.push('Return JSON only, in this exact shape:'); - lines.push('{"scores":['); - lines.push(' {"output":1,"correctness":N,"completeness":N,"code_quality":N,"edge_cases":N,"overall":N,"notes":"..."},'); - lines.push(' ...'); - lines.push(']}'); - lines.push(''); - lines.push('overall = rounded average of the 4 dimensions. No other commentary.'); - return lines.join('\n'); -} - -interface ParsedScore { - overall: number; - dimensions: Record; -} - -function parseScores(raw: string, expectedCount: number): ParsedScore[] { - const match = raw.match(/\{[\s\S]*\}/); - if (!match) return []; - try { - const obj = JSON.parse(match[0]); - if (!Array.isArray(obj.scores)) return []; - return obj.scores.slice(0, expectedCount).map((s: Record) => ({ - overall: Number(s.overall ?? 0), - dimensions: { - correctness: Number(s.correctness ?? 0), - completeness: Number(s.completeness ?? 0), - code_quality: Number(s.code_quality ?? 0), - edge_cases: Number(s.edge_cases ?? 0), - }, - })); - } catch { - return []; - } -} diff --git a/lib/model-benchmark/runner.ts b/lib/model-benchmark/runner.ts deleted file mode 100644 index cbef4107b..000000000 --- a/lib/model-benchmark/runner.ts +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Multi-provider benchmark runner. - * - * Orchestrates running the same prompt across multiple provider adapters and - * aggregates RunResult outputs + judge scores into a single report. Adapters - * run in parallel (Promise.allSettled) so a slow provider doesn't block a fast - * one. Per-provider auth/timeout/rate-limit errors don't abort the batch. - */ - -import type { ProviderAdapter, RunOpts, RunResult } from './providers/types'; -import { ClaudeAdapter } from './providers/claude'; -import { GptAdapter } from './providers/gpt'; -import { GeminiAdapter } from './providers/gemini'; - -export interface BenchmarkInput { - prompt: string; - workdir: string; - timeoutMs?: number; - /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */ - providers: Array<'claude' | 'gpt' | 'gemini'>; - /** Optional per-provider model overrides. */ - models?: Partial>; - /** If true, skip providers whose available() returns !ok. If false, include them with error. */ - skipUnavailable?: boolean; -} - -export interface BenchmarkEntry { - provider: string; - family: 'claude' | 'gpt' | 'gemini'; - available: boolean; - unavailable_reason?: string; - result?: RunResult; - costUsd?: number; - /** Judge score 0-10 across dimensions. Populated separately by the judge step. */ - qualityScore?: number; - qualityDetails?: Record; -} - -export interface BenchmarkReport { - prompt: string; - workdir: string; - startedAt: string; - durationMs: number; - entries: BenchmarkEntry[]; -} - -const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = { - claude: () => new ClaudeAdapter(), - gpt: () => new GptAdapter(), - gemini: () => new GeminiAdapter(), -}; - -export async function runBenchmark(input: BenchmarkInput): Promise { - const startedAtMs = Date.now(); - const startedAt = new Date(startedAtMs).toISOString(); - const timeoutMs = input.timeoutMs ?? 300_000; - - const entries: BenchmarkEntry[] = []; - const runPromises: Array> = []; - - for (const name of input.providers) { - const factory = ADAPTERS[name]; - if (!factory) { - entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` }); - continue; - } - const adapter = factory(); - const entry: BenchmarkEntry = { provider: adapter.name, family: adapter.family, available: true }; - entries.push(entry); - - runPromises.push((async () => { - const check = await adapter.available(); - entry.available = check.ok; - if (!check.ok) { - entry.unavailable_reason = check.reason; - if (input.skipUnavailable) return; - } - const opts: RunOpts = { - prompt: input.prompt, - workdir: input.workdir, - timeoutMs, - model: input.models?.[name], - }; - const res = await adapter.run(opts); - entry.result = res; - entry.costUsd = adapter.estimateCost(res.tokens, res.modelUsed); - })()); - } - - await Promise.allSettled(runPromises); - - return { - prompt: input.prompt, - workdir: input.workdir, - startedAt, - durationMs: Date.now() - startedAtMs, - entries, - }; -} - -export function formatTable(report: BenchmarkReport): string { - const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`; - const sep = '-'.repeat(header.length); - const rows: string[] = [header, sep]; - for (const e of report.entries) { - if (!e.available) { - rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`); - continue; - } - const r = e.result!; - if (r.error) { - rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`); - continue; - } - const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-'; - rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`); - } - return rows.join('\n'); -} - -export function formatJson(report: BenchmarkReport): string { - return JSON.stringify(report, null, 2); -} - -export function formatMarkdown(report: BenchmarkReport): string { - const lines: string[] = [ - `# Benchmark report — ${report.startedAt}`, - '', - `**Prompt:** ${report.prompt.length > 200 ? report.prompt.slice(0, 200) + '…' : report.prompt}`, - `**Workdir:** \`${report.workdir}\``, - `**Total duration:** ${msToStr(report.durationMs)}`, - '', - '| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |', - '|-------|---------|-----------------|------|---------|-------|-------|', - ]; - for (const e of report.entries) { - if (!e.available) { - lines.push(`| ${e.provider} | - | - | - | - | - | unavailable: ${e.unavailable_reason ?? 'unknown'} |`); - continue; - } - const r = e.result!; - if (r.error) { - lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`); - continue; - } - const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-'; - lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`); - } - return lines.join('\n'); -} - -function pad(s: string, n: number): string { - return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length); -} - -function msToStr(ms: number): string { - if (ms < 1000) return `${ms}ms`; - return `${(ms / 1000).toFixed(1)}s`; -} - -function fmtCost(usd?: number): string { - if (usd === undefined) return '-'; - if (usd < 0.01) return `$${usd.toFixed(4)}`; - return `$${usd.toFixed(2)}`; -} diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index 8edea3b24..0376831b5 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -1,7 +1,7 @@ /** * gstack-model-benchmark CLI tests (offline). * - * Covers CLI wiring that unit tests against benchmark-runner.ts can't see: + * Covers CLI wiring that unit tests can't see: * - --dry-run auth/provider-list resolution * - unknown provider WARN path * - provider default (claude) when --models omitted @@ -66,11 +66,22 @@ describe('gstack-model-benchmark --dry-run', () => { expect(r.stdout).toContain('providers: claude'); }); - test('--timeout-ms and --workdir flags flow through to dry-run report', () => { - const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--workdir', '/tmp', '--dry-run']); + test('--timeout-ms flows through to dry-run report', () => { + const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--dry-run']); expect(r.status).toBe(0); expect(r.stdout).toContain('timeout_ms: 9999'); - expect(r.stdout).toContain('workdir: /tmp'); + }); + + test('no prompt falls back to the corpus (not an error)', () => { + const r = run(['--models', 'claude', '--dry-run']); + expect(r.status).toBe(0); + expect(r.stdout).toMatch(/corpus:\s+\d+ case/); + }); + + test('upload is off by default (local only, nothing uploaded)', () => { + const r = run(['--prompt', 'hi', '--dry-run'], { env: { BRAINTRUST_API_KEY: '' } }); + expect(r.status).toBe(0); + expect(r.stdout).toContain('local only'); }); test('--judge flag reported in dry-run output', () => { @@ -196,9 +207,9 @@ describe('gstack-model-benchmark prompt resolution', () => { expect(r.stdout).toContain('treat-me-as-inline'); }); - test('missing prompt exits non-zero', () => { + test('no positional and no --prompt runs the corpus', () => { const r = run(['--dry-run']); - expect(r.status).not.toBe(0); - expect(r.stderr).toContain('specify a prompt'); + expect(r.status).toBe(0); + expect(r.stdout).toMatch(/corpus:\s+\d+ case/); }); }); diff --git a/test/benchmark-production-boundary.test.ts b/test/benchmark-production-boundary.test.ts index 672664e65..dca91488f 100644 --- a/test/benchmark-production-boundary.test.ts +++ b/test/benchmark-production-boundary.test.ts @@ -42,12 +42,8 @@ test('production modules do not import from test directories', () => { test('former test-helper paths re-export the production benchmark API', async () => { const [ - runner, - helperRunner, pricing, helperPricing, - judge, - helperJudge, claude, helperClaude, gpt, @@ -55,12 +51,8 @@ test('former test-helper paths re-export the production benchmark API', async () gemini, helperGemini, ] = await Promise.all([ - import('../lib/model-benchmark/runner'), - import('./helpers/benchmark-runner'), import('../lib/model-benchmark/pricing'), import('./helpers/pricing'), - import('../lib/model-benchmark/judge'), - import('./helpers/benchmark-judge'), import('../lib/model-benchmark/providers/claude'), import('./helpers/providers/claude'), import('../lib/model-benchmark/providers/gpt'), @@ -69,9 +61,7 @@ test('former test-helper paths re-export the production benchmark API', async () import('./helpers/providers/gemini'), ]); - expect(helperRunner.runBenchmark).toBe(runner.runBenchmark); expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd); - expect(helperJudge.judgeEntries).toBe(judge.judgeEntries); expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter); expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter); expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter); diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index 0e62edd64..fb4664d14 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -1,17 +1,12 @@ /** - * Unit tests for the benchmark runner. + * Unit tests for benchmark pricing + tool-compatibility helpers. * - * Mocks adapters to verify: - * - All adapters run in parallel (Promise.allSettled not serial) - * - Unavailable adapters are skipped or marked depending on flag - * - Per-adapter errors don't abort the batch - * - Output formatters (table, json, markdown) produce non-empty strings - * - * Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those. + * Orchestration, scoring, and reporting moved to Braintrust + * (lib/model-benchmark/braintrust-eval.ts); those are covered by + * model-benchmark-braintrust.test.ts and the live e2e suite. */ import { test, expect } from 'bun:test'; -import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from '../lib/model-benchmark/runner'; import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing'; import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map'; @@ -58,80 +53,3 @@ test('TOOL_COMPATIBILITY is populated for all three families', () => { expect(TOOL_COMPATIBILITY.gpt).toBeDefined(); expect(TOOL_COMPATIBILITY.gemini).toBeDefined(); }); - -test('formatTable handles a report with mixed success/error/unavailable entries', () => { - const report: BenchmarkReport = { - prompt: 'test prompt', - workdir: '/tmp', - startedAt: '2026-04-16T20:00:00Z', - durationMs: 1500, - entries: [ - { - provider: 'claude', - family: 'claude', - available: true, - result: { - output: 'ok', - tokens: { input: 100, output: 200 }, - durationMs: 800, - toolCalls: 3, - modelUsed: 'claude-opus-4-7', - }, - costUsd: 0.0165, - qualityScore: 9.2, - }, - { - provider: 'gpt', - family: 'gpt', - available: true, - result: { - output: '', - tokens: { input: 0, output: 0 }, - durationMs: 200, - toolCalls: 0, - modelUsed: 'gpt-5.4', - error: { code: 'auth', reason: 'codex login required' }, - }, - }, - { - provider: 'gemini', - family: 'gemini', - available: false, - unavailable_reason: 'gemini CLI not on PATH', - }, - ], - }; - - const table = formatTable(report); - expect(table).toContain('claude-opus-4-7'); - expect(table).toContain('ERROR auth'); - expect(table).toContain('unavailable'); - expect(table).toContain('9.2/10'); -}); - -test('formatJson produces parseable JSON', () => { - const report: BenchmarkReport = { - prompt: 'x', - workdir: '/tmp', - startedAt: '2026-04-16T20:00:00Z', - durationMs: 100, - entries: [], - }; - const json = formatJson(report); - const parsed = JSON.parse(json); - expect(parsed.prompt).toBe('x'); - expect(parsed.entries).toEqual([]); -}); - -test('formatMarkdown produces a table header', () => { - const report: BenchmarkReport = { - prompt: 'x', - workdir: '/tmp', - startedAt: '2026-04-16T20:00:00Z', - durationMs: 100, - entries: [], - }; - const md = formatMarkdown(report); - expect(md).toContain('# Benchmark report'); - expect(md).toContain('| Model | Latency |'); -}); diff --git a/test/helpers/benchmark-judge.ts b/test/helpers/benchmark-judge.ts deleted file mode 100644 index 0da01d276..000000000 --- a/test/helpers/benchmark-judge.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Compatibility export for tests and downstream tooling that used the former helper path. -export * from '../../lib/model-benchmark/judge'; diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts deleted file mode 100644 index 585a89d3e..000000000 --- a/test/helpers/benchmark-runner.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Compatibility export for tests and downstream tooling that used the former helper path. -export * from '../../lib/model-benchmark/runner'; diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index cccd82ddc..2eaabc0ec 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -14,7 +14,7 @@ * - Parallel execution via Promise.allSettled — slow provider doesn't block fast * * NOT covered here (would need dedicated test files): - * - Quality judge integration (benchmark-judge.ts, adds ~$0.05/run) + * - Quality judge integration (autoevals ClosedQA, opt-in) * - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0` */ @@ -22,7 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude'; import { GptAdapter } from '../lib/model-benchmark/providers/gpt'; import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini'; -import { runBenchmark } from '../lib/model-benchmark/runner'; +import { runProviderBenchmark } from '../lib/model-benchmark/braintrust-eval'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -163,30 +163,23 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(result.durationMs).toBeGreaterThan(0); }, 30_000); - test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => { - // Use the full runner with all three providers — whichever are unauthed should - // return entries with available=false and not crash the batch. - const report = await runBenchmark({ - prompt: PROMPT, - workdir, - providers: ['claude', 'gpt', 'gemini'], - timeoutMs: 120_000, - skipUnavailable: false, - }); - expect(report.entries).toHaveLength(3); - for (const e of report.entries) { - expect(['claude', 'gpt', 'gemini']).toContain(e.family); - if (e.available) { - expect(e.result).toBeDefined(); - } else { - expect(typeof e.unavailable_reason).toBe('string'); - } + test('runProviderBenchmark: an unauthed/failing provider returns a result, never throws', async () => { + // Braintrust owns orchestration now. The property we care about: a provider + // that's unavailable or errors comes back as a ProviderBenchmark (score null, + // ops carrying the error) instead of throwing and aborting the batch. + const cases = [{ id: 'smoke', input: PROMPT, required: ['ok'] }]; + const results = await Promise.all( + (['claude', 'gpt', 'gemini'] as const).map(p => runProviderBenchmark(p, cases, { timeoutMs: 120_000 })), + ); + expect(results).toHaveLength(3); + for (const r of results) { + expect(['claude', 'gpt', 'gemini']).toContain(r.provider); + expect(r.score === null || (typeof r.score === 'number' && r.score >= 0 && r.score <= 1)).toBe(true); + expect(Array.isArray(r.ops)).toBe(true); } - // At least one available provider should have produced a non-error result in a healthy CI env. - const hadSuccess = report.entries.some(e => e.available && e.result && !e.result.error); - // We don't hard-assert this: if NO providers are authed, skip silently. + const hadSuccess = results.some(r => typeof r.score === 'number' && r.ops.some(o => !o.error)); if (!hadSuccess) { - process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n'); + process.stderr.write('\nbenchmark live: no provider produced a clean result (no auth?)\n'); } }, 300_000); });