feat: back model benchmark with Braintrust, drop in-house scoring

Braintrust now owns benchmark scoring, experiments, comparison, and
reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim)
plus operational metrics the CLIs report. runProviderBenchmark wraps each
adapter as a Braintrust Eval task; a deterministic required-terms scorer
replaces the in-house evaluation logic and the optional autoevals ClosedQA
judge replaces judge.ts.

Runs local by default under bun: with no BRAINTRUST_API_KEY it sets
noSendLogs and ships nothing; setting the key opts into the cloud dashboard.
An empty output with zero tokens is thrown so a silent auth/CLI failure
can't masquerade as a 0.0 score.

Deletes runner.ts and judge.ts (and their test-helper re-export shims);
rewires bin/gstack-model-benchmark and adapts the benchmark tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

bin/gstack-model-benchmark

@@ -1,33 +1,37 @@
 #!/usr/bin/env bun
 /**
- * gstack-model-benchmark — run the same prompt across multiple providers
+ * gstack-model-benchmark — run the same prompt(s) across providers and compare,
- * and compare latency, tokens, cost, quality, and tool-call count.
+ * with scoring, experiments, and reporting owned by Braintrust (local by default;
+ * nothing is uploaded unless BRAINTRUST_API_KEY is set — that key is the consent
+ * boundary for the cloud dashboard).
  *
  * Usage:
- *   gstack-model-benchmark <skill-or-prompt-file> [options]
+ *   gstack-model-benchmark [<prompt-file>] [options]
  *
  * Options:
  *   --models claude,gpt,gemini   Comma-separated provider list (default: claude)
- *   --prompt "<text>"            Inline prompt instead of a file
+ *   --prompt "<text>"            Ad-hoc single-case prompt instead of the corpus
- *   --workdir <path>             Working dir passed to each CLI (default: cwd)
+ *   --corpus <path>              Corpus JSON (default: evals/model-benchmark/corpus.json)
- *   --timeout-ms <n>             Per-provider timeout (default: 300000)
+ *   --timeout-ms <n>             Per-provider-per-case timeout (default: 300000)
- *   --output table|json|markdown Output format (default: table)
+ *   --output table|json          Output format (default: table)
- *   --skip-unavailable           Skip providers that fail available() check
+ *   --judge                      Add the autoevals ClosedQA LLM judge (needs OPENAI_API_KEY)
- *                                (default: include them with unavailable marker)
- *   --judge                      Run Anthropic SDK judge on outputs for quality score
- *                                (requires ANTHROPIC_API_KEY; adds ~$0.05 per call)
  *   --dry-run                    Validate flags + resolve auth, don't invoke providers
  *
  * Examples:
  *   gstack-model-benchmark --prompt "Write a haiku about databases" --models claude,gpt
- *   gstack-model-benchmark ./test-prompt.txt --models claude,gpt,gemini --judge
+ *   gstack-model-benchmark --models claude,gpt,gemini            # runs the corpus
  *   gstack-model-benchmark --prompt "hi" --models claude,gpt,gemini --dry-run
  */
 
 import '../lib/conductor-env-shim';
 import * as fs from 'fs';
-import * as path from 'path';
+import {
-import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkInput } from '../lib/model-benchmark/runner';
+  loadCorpus,
+  runProviderBenchmark,
+  type BenchCase,
+  type ProviderBenchmark,
+  type ProviderName,
+} from '../lib/model-benchmark/braintrust-eval';
 import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
 import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
 import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';
@@ -38,10 +42,10 @@ const ADAPTER_FACTORIES = {
   gemini: () => new GeminiAdapter(),
 };
 
-type OutputFormat = 'table' | 'json' | 'markdown';
+type OutputFormat = 'table' | 'json';
 
 const CLI_ARGS = process.argv.slice(2);
-const VALUE_FLAGS = new Set(['--models', '--prompt', '--workdir', '--timeout-ms', '--output']);
+const VALUE_FLAGS = new Set(['--models', '--prompt', '--corpus', '--timeout-ms', '--output']);
 
 function arg(name: string, def?: string): string | undefined {
   const idx = CLI_ARGS.findIndex(a => a === name || a.startsWith(name + '='));
@@ -76,93 +80,92 @@ function positionalArgs(args: string[]): string[] {
   return positional;
 }
 
-function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> {
+function parseProviders(s: string | undefined): ProviderName[] {
   if (!s) return ['claude'];
-  const seen = new Set<'claude' | 'gpt' | 'gemini'>();
+  const seen = new Set<ProviderName>();
   for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) {
     if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p);
-    else {
+    else console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`);
-      console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`);
-    }
   }
   return seen.size ? Array.from(seen) : ['claude'];
 }
 
-function resolvePrompt(positional: string | undefined): string {
+/** Resolve the cases: --prompt/file → single ad-hoc case; else the corpus. */
+function resolveCases(positional: string | undefined): BenchCase[] {
   const inline = arg('--prompt');
-  if (inline) return inline;
+  if (inline) return [{ id: 'adhoc', input: inline, required: [] }];
-  if (!positional) {
+  if (positional && fs.existsSync(positional)) {
-    console.error('ERROR: specify a prompt via positional path or --prompt "<text>"');
+    return [{ id: 'adhoc', input: fs.readFileSync(positional, 'utf-8'), required: [] }];
-    process.exit(1);
   }
-  if (fs.existsSync(positional)) {
+  if (positional) return [{ id: 'adhoc', input: positional, required: [] }];
-    return fs.readFileSync(positional, 'utf-8');
+  return loadCorpus(arg('--corpus'));
-  }
-  // Not a file — treat as inline prompt
-  return positional;
 }
 
 async function main(): Promise<void> {
   const positional = positionalArgs(CLI_ARGS)[0];
-  const prompt = resolvePrompt(positional);
   const providers = parseProviders(arg('--models'));
-  const workdir = arg('--workdir', process.cwd())!;
   const timeoutMs = parseInt(arg('--timeout-ms', '300000')!, 10);
   const output = (arg('--output', 'table') as OutputFormat);
-  const skipUnavailable = flag('--skip-unavailable');
   const doJudge = flag('--judge');
   const dryRun = flag('--dry-run');
 
   if (dryRun) {
-    await dryRunReport({ prompt, providers, workdir, timeoutMs, output, doJudge });
+    await dryRunReport({ cases: resolveCases(positional), providers, timeoutMs, output, doJudge });
     return;
   }
 
-  const input: BenchmarkInput = {
+  if (doJudge && !process.env.OPENAI_API_KEY) {
-    prompt,
+    console.error('WARN: --judge needs OPENAI_API_KEY for the autoevals ClosedQA judge — running with the deterministic scorer only.');
-    workdir,
-    providers,
-    timeoutMs,
-    skipUnavailable,
-  };
-
-  const report = await runBenchmark(input);
-
-  if (doJudge) {
-    try {
-      const { judgeEntries } = await import('../lib/model-benchmark/judge');
-      await judgeEntries(report);
-    } catch (err) {
-      console.error(`WARN: judge unavailable: ${(err as Error).message}`);
-    }
   }
 
-  let out: string;
+  const cases = resolveCases(positional);
-  switch (output) {
+  const results: ProviderBenchmark[] = [];
-    case 'json':     out = formatJson(report); break;
+  for (const provider of providers) {
-    case 'markdown': out = formatMarkdown(report); break;
+    results.push(await runProviderBenchmark(provider, cases, { timeoutMs, judge: doJudge && !!process.env.OPENAI_API_KEY }));
-    case 'table':
-    default:         out = formatTable(report); break;
   }
-  process.stdout.write(out + '\n');
+
+  process.stdout.write((output === 'json' ? JSON.stringify(results, null, 2) : formatTable(results)) + '\n');
+}
+
+function formatTable(results: ProviderBenchmark[]): string {
+  const header = `Provider   Score    Cases  Latency(avg)  Tokens(in→out)  Cost      Errors`;
+  const rows: string[] = [header, '-'.repeat(header.length)];
+  for (const r of results) {
+    const n = r.ops.length || 1;
+    const avgMs = r.ops.reduce((s, o) => s + o.durationMs, 0) / n;
+    const tin = r.ops.reduce((s, o) => s + o.tokensIn, 0);
+    const tout = r.ops.reduce((s, o) => s + o.tokensOut, 0);
+    const cost = r.ops.reduce((s, o) => s + o.costUsd, 0);
+    const errs = r.ops.filter(o => o.error).length;
+    const score = r.score === null ? '—' : `${(r.score * 100).toFixed(1)}%`;
+    rows.push(
+      `${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${pad(`${tin}→${tout}`, 15)} ${pad(fmtCost(cost), 9)} ${errs || ''}`,
+    );
+  }
+  return rows.join('\n');
 }
 
 async function dryRunReport(opts: {
-  prompt: string;
+  cases: BenchCase[];
-  providers: Array<'claude' | 'gpt' | 'gemini'>;
+  providers: ProviderName[];
-  workdir: string;
   timeoutMs: number;
   output: OutputFormat;
   doJudge: boolean;
 }): Promise<void> {
+  const adhoc = opts.cases.length === 1 && opts.cases[0].id === 'adhoc';
   const lines: string[] = [];
   lines.push('== gstack-model-benchmark --dry-run ==');
-  lines.push(`  prompt:     ${opts.prompt.length > 80 ? opts.prompt.slice(0, 80) + '…' : opts.prompt}`);
+  if (adhoc) {
+    const p = opts.cases[0].input;
+    lines.push(`  prompt:     ${p.length > 80 ? p.slice(0, 80) + '…' : p}`);
+  } else {
+    lines.push(`  corpus:     ${opts.cases.length} case(s)`);
+  }
   lines.push(`  providers:  ${opts.providers.join(', ')}`);
-  lines.push(`  workdir:    ${opts.workdir}`);
   lines.push(`  timeout_ms: ${opts.timeoutMs}`);
   lines.push(`  output:     ${opts.output}`);
-  lines.push(`  judge:      ${opts.doJudge ? 'on (Anthropic SDK)' : 'off'}`);
+  lines.push(`  judge:      ${opts.doJudge ? 'on (autoevals ClosedQA)' : 'off'}`);
+  lines.push(`  upload:     ${process.env.BRAINTRUST_API_KEY ? 'ON — BRAINTRUST_API_KEY set (cloud dashboard)' : 'off (local only, nothing uploaded)'}`);
   lines.push('');
   lines.push('Adapter availability:');
   let authFailures = 0;
@@ -173,20 +176,28 @@ async function dryRunReport(opts: {
       authFailures += 1;
       continue;
     }
-    const adapter = factory();
+    const check = await factory().available();
-    const check = await adapter.available();
+    if (check.ok) lines.push(`  ${name}: OK`);
-    if (check.ok) {
+    else { lines.push(`  ${name}: NOT READY — ${check.reason}`); authFailures += 1; }
-      lines.push(`  ${adapter.name}: OK`);
-    } else {
-      lines.push(`  ${adapter.name}: NOT READY — ${check.reason}`);
-      authFailures += 1;
-    }
   }
   lines.push('');
   lines.push(`(--dry-run — no prompts sent. ${authFailures} provider(s) unavailable.)`);
   process.stdout.write(lines.join('\n') + '\n');
 }
 
+function pad(s: string, n: number): string {
+  return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
+}
+function msToStr(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  return `${(ms / 1000).toFixed(1)}s`;
+}
+function fmtCost(usd: number): string {
+  if (usd === 0) return '$0';
+  if (usd < 0.01) return `$${usd.toFixed(4)}`;
+  return `$${usd.toFixed(2)}`;
+}
+
 main().catch(err => {
   console.error('FATAL:', err);
   process.exit(1);

evals/model-benchmark/corpus.json

@@ -0,0 +1,17 @@
+[
+  {
+    "id": "security-review",
+    "input": "Review a synthetic patch containing command injection, unsafe recursive delete, and path traversal. Do not mutate files. Report only.",
+    "required": ["command injection", "recursive delete", "path traversal", "no mutation"]
+  },
+  {
+    "id": "graph-indexer-consent",
+    "input": "Decide whether GStack should require a third-party graph indexer. Justify the install policy.",
+    "required": ["optional", "consent", "fallback"]
+  },
+  {
+    "id": "resolver-safety",
+    "input": "Diagnose a resolver that passes an untrusted logical skill name into path.resolve. Recommend the fix.",
+    "required": ["allowlist", "single segment", "traversal"]
+  }
+]

evals/model-benchmark/gstack.eval.ts

@@ -0,0 +1,21 @@
+/**
+ * GStack model benchmark — Braintrust entrypoint for `bun run` / CI.
+ *
+ * Braintrust owns scoring, comparison, and reporting. See
+ * lib/model-benchmark/braintrust-eval.ts for the core; the user-facing CLI is
+ * bin/gstack-model-benchmark.
+ *
+ * Local run, nothing leaves the machine:
+ *   GSTACK_BENCH_PROVIDER=claude bun run evals/model-benchmark/gstack.eval.ts
+ *
+ * Opt into the Braintrust dashboard (consent-gated cloud):
+ *   export BRAINTRUST_API_KEY=...
+ *   for p in claude gpt gemini; do GSTACK_BENCH_PROVIDER=$p bun run evals/model-benchmark/gstack.eval.ts; done
+ */
+import { loadCorpus, runProviderBenchmark, type ProviderName } from '../../lib/model-benchmark/braintrust-eval';
+
+const provider = (process.env.GSTACK_BENCH_PROVIDER ?? 'claude') as ProviderName;
+const timeoutMs = Number(process.env.GSTACK_BENCH_TIMEOUT_MS ?? 300_000);
+const judge = process.env.GSTACK_BENCH_JUDGE === '1';
+
+await runProviderBenchmark(provider, loadCorpus(), { timeoutMs, judge });

lib/model-benchmark/braintrust-eval.ts

@@ -0,0 +1,160 @@
+/**
+ * Braintrust-backed benchmark core.
+ *
+ * Braintrust owns scoring, experiment tracking, comparison, and reporting. GStack
+ * keeps only the unavoidable CLI-agent shims (providers/*.ts) as the eval `task`,
+ * plus operational metrics (latency/tokens/cost) the CLIs report and Braintrust
+ * doesn't measure for us.
+ *
+ * Local by default: with no BRAINTRUST_API_KEY, runs with `noSendLogs` and ships
+ * nothing. Setting the key opts into the cloud dashboard (the consent boundary).
+ * Runs under bun (the adapters use Bun.which), so invoke via `bun run`, never the
+ * node-based `braintrust eval` CLI.
+ */
+import { Eval } from 'braintrust';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { ClaudeAdapter } from './providers/claude';
+import { GptAdapter } from './providers/gpt';
+import { GeminiAdapter } from './providers/gemini';
+import type { ProviderAdapter } from './providers/types';
+
+export type ProviderName = 'claude' | 'gpt' | 'gemini';
+
+const ADAPTERS: Record<ProviderName, () => ProviderAdapter> = {
+  claude: () => new ClaudeAdapter(),
+  gpt: () => new GptAdapter(),
+  gemini: () => new GeminiAdapter(),
+};
+
+export interface BenchCase {
+  id: string;
+  input: string;
+  required: string[];
+}
+
+export interface BenchOpts {
+  timeoutMs?: number;
+  /** Add an autoevals LLM judge (ClosedQA). Needs OPENAI_API_KEY. */
+  judge?: boolean;
+}
+
+/** Per-case operational metrics the adapter reports — Braintrust doesn't time the CLI for us. */
+export interface CaseOps {
+  id: string;
+  durationMs: number;
+  tokensIn: number;
+  tokensOut: number;
+  costUsd: number;
+  toolCalls: number;
+  modelUsed: string;
+  error?: string;
+}
+
+export interface ProviderBenchmark {
+  provider: ProviderName;
+  /** Mean required-terms score 0..1 from Braintrust, or null if every case errored. */
+  score: number | null;
+  ops: CaseOps[];
+}
+
+export const DEFAULT_CORPUS_PATH = path.join(__dirname, '..', '..', 'evals', 'model-benchmark', 'corpus.json');
+
+export function loadCorpus(corpusPath = DEFAULT_CORPUS_PATH): BenchCase[] {
+  return JSON.parse(fs.readFileSync(corpusPath, 'utf-8'));
+}
+
+interface ScorerArgs {
+  input: string;
+  output: string;
+  expected: string[];
+}
+type ScoreResult = { name: string; score: number };
+type Scorer = (args: ScorerArgs) => ScoreResult | Promise<ScoreResult>;
+
+/** Deterministic scorer: fraction of required terms present. Replaces the old in-house evaluation.ts. */
+const requiredTerms: Scorer = ({ output, expected }) => {
+  const norm = (output ?? '').toLowerCase();
+  const matched = (expected ?? []).filter((t) => norm.includes(t.toLowerCase()));
+  return { name: 'required-terms', score: expected?.length ? matched.length / expected.length : 1 };
+};
+
+/**
+ * Run one provider across the cases through Braintrust and return its score + ops.
+ * The adapter is the `task`; requiredTerms (and optionally an autoevals judge) are
+ * the `scores`. Empty output with zero tokens is thrown so it can't fake a 0.
+ */
+export async function runProviderBenchmark(
+  provider: ProviderName,
+  cases: BenchCase[],
+  opts: BenchOpts = {},
+): Promise<ProviderBenchmark> {
+  const factory = ADAPTERS[provider];
+  if (!factory) throw new Error(`unknown provider '${provider}' (claude|gpt|gemini)`);
+  const adapter = factory();
+  const timeoutMs = opts.timeoutMs ?? 300_000;
+
+  const ops: CaseOps[] = [];
+  const byInput = new Map(cases.map((c) => [c.input, c]));
+
+  // Braintrust calls scorers with { input, output, expected, metadata }.
+  const scores: Scorer[] = [requiredTerms];
+  if (opts.judge) {
+    // autoevals ClosedQA = Braintrust's own LLM-judge, replacing the in-house judge.ts.
+    // Normalize its result to a plain { name, score } so cross-package Score types don't clash.
+    const { ClosedQA } = await import('autoevals');
+    const closedQa = ClosedQA as unknown as (a: Record<string, unknown>) => Promise<{ score?: number }>;
+    scores.push(async ({ input, output, expected }) => {
+      const r = await closedQa({
+        input,
+        output,
+        criteria: `Addresses the task and mentions: ${(expected ?? []).join(', ')}`,
+      });
+      return { name: 'judge-closedqa', score: typeof r?.score === 'number' ? r.score : 0 };
+    });
+  }
+
+  const noSendLogs = !process.env.BRAINTRUST_API_KEY;
+
+  const result = await Eval(
+    `gstack-model-benchmark:${provider}`,
+    {
+      data: () => cases.map((c) => ({ input: c.input, expected: c.required, metadata: { id: c.id } })),
+      task: async (input: string) => {
+        const c = byInput.get(input);
+        const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-'));
+        try {
+          const r = await adapter.run({ prompt: input, workdir, timeoutMs });
+          const costUsd = adapter.estimateCost(r.tokens, r.modelUsed);
+          const op: CaseOps = {
+            id: c?.id ?? input.slice(0, 24),
+            durationMs: r.durationMs,
+            tokensIn: r.tokens.input,
+            tokensOut: r.tokens.output,
+            costUsd,
+            toolCalls: r.toolCalls,
+            modelUsed: r.modelUsed,
+            error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined,
+          };
+          ops.push(op);
+          if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`);
+          // Empty output + zero tokens = provider never answered (silent auth/CLI failure).
+          // Fail loud so it can't masquerade as a 0.0 score.
+          if (!r.output.trim() && r.tokens.input === 0 && r.tokens.output === 0) {
+            throw new Error(`${provider} returned empty output with zero tokens (likely auth/CLI failure, not a real result)`);
+          }
+          return r.output;
+        } finally {
+          fs.rmSync(workdir, { recursive: true, force: true });
+        }
+      },
+      scores,
+    },
+    { noSendLogs },
+  );
+
+  const scoreSummary = result.summary?.scores?.['required-terms'];
+  const score = typeof scoreSummary?.score === 'number' ? scoreSummary.score : null;
+  return { provider, score, ops };
+}

lib/model-benchmark/judge.ts

@@ -1,101 +0,0 @@
-/**
- * Benchmark quality judge for multi-provider scoring.
- *
- * The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
- * the prompt + N provider outputs and scores each on: correctness, completeness,
- * code quality, edge case handling. 0-10 per dimension; overall = average.
- *
- * Judge adds ~$0.05 per benchmark run. Gated by --judge CLI flag.
- */
-
-import type { BenchmarkReport, BenchmarkEntry } from './runner';
-
-export async function judgeEntries(report: BenchmarkReport): Promise<void> {
-  if (!process.env.ANTHROPIC_API_KEY) {
-    throw new Error('ANTHROPIC_API_KEY not set — judge requires Anthropic access.');
-  }
-  const { default: Anthropic } = await import('@anthropic-ai/sdk').catch(() => {
-    throw new Error('@anthropic-ai/sdk not installed — run `bun add @anthropic-ai/sdk` if you want the judge.');
-  });
-  const client = new (Anthropic as unknown as new (opts: { apiKey: string }) => {
-    messages: { create: (params: Record<string, unknown>) => Promise<{ content: Array<{ type: string; text: string }> }> };
-  })({ apiKey: process.env.ANTHROPIC_API_KEY! });
-
-  const successful = report.entries.filter(e => e.available && e.result && !e.result.error);
-  if (successful.length === 0) return;
-
-  const judgePrompt = buildJudgePrompt(report.prompt, successful);
-  const msg = await client.messages.create({
-    model: 'claude-sonnet-4-6',
-    max_tokens: 2048,
-    messages: [{ role: 'user', content: judgePrompt }],
-  });
-  const textBlock = msg.content.find(c => c.type === 'text');
-  if (!textBlock) return;
-
-  const scores = parseScores(textBlock.text, successful.length);
-  for (let i = 0; i < successful.length; i++) {
-    const s = scores[i];
-    if (!s) continue;
-    successful[i].qualityScore = s.overall;
-    successful[i].qualityDetails = s.dimensions;
-  }
-}
-
-function buildJudgePrompt(prompt: string, entries: BenchmarkEntry[]): string {
-  const lines: string[] = [
-    'You are a strict, fair technical reviewer scoring N model outputs against the same prompt.',
-    '',
-    '--- PROMPT ---',
-    prompt.length > 4000 ? prompt.slice(0, 4000) + '\n[...truncated for judge budget...]' : prompt,
-    '',
-    '--- OUTPUTS ---',
-  ];
-  entries.forEach((e, i) => {
-    const r = e.result!;
-    const out = r.output.length > 3000 ? r.output.slice(0, 3000) + '\n[...truncated...]' : r.output;
-    lines.push(`=== Output ${i + 1}: ${r.modelUsed} ===`);
-    lines.push(out);
-    lines.push('');
-  });
-  lines.push('');
-  lines.push('Score each output on these dimensions (0-10 per dimension):');
-  lines.push('  - correctness:   does it solve what the prompt asked?');
-  lines.push('  - completeness:  are edge cases and error paths addressed?');
-  lines.push('  - code_quality:  naming, structure, explicitness');
-  lines.push('  - edge_cases:    handling of nil/empty/invalid input');
-  lines.push('');
-  lines.push('Return JSON only, in this exact shape:');
-  lines.push('{"scores":[');
-  lines.push('  {"output":1,"correctness":N,"completeness":N,"code_quality":N,"edge_cases":N,"overall":N,"notes":"..."},');
-  lines.push('  ...');
-  lines.push(']}');
-  lines.push('');
-  lines.push('overall = rounded average of the 4 dimensions. No other commentary.');
-  return lines.join('\n');
-}
-
-interface ParsedScore {
-  overall: number;
-  dimensions: Record<string, number>;
-}
-
-function parseScores(raw: string, expectedCount: number): ParsedScore[] {
-  const match = raw.match(/\{[\s\S]*\}/);
-  if (!match) return [];
-  try {
-    const obj = JSON.parse(match[0]);
-    if (!Array.isArray(obj.scores)) return [];
-    return obj.scores.slice(0, expectedCount).map((s: Record<string, number>) => ({
-      overall: Number(s.overall ?? 0),
-      dimensions: {
-        correctness: Number(s.correctness ?? 0),
-        completeness: Number(s.completeness ?? 0),
-        code_quality: Number(s.code_quality ?? 0),
-        edge_cases: Number(s.edge_cases ?? 0),
-      },
-    }));
-  } catch {
-    return [];
-  }
-}

lib/model-benchmark/runner.ts

@@ -1,165 +0,0 @@
-/**
- * Multi-provider benchmark runner.
- *
- * Orchestrates running the same prompt across multiple provider adapters and
- * aggregates RunResult outputs + judge scores into a single report. Adapters
- * run in parallel (Promise.allSettled) so a slow provider doesn't block a fast
- * one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
- */
-
-import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
-import { ClaudeAdapter } from './providers/claude';
-import { GptAdapter } from './providers/gpt';
-import { GeminiAdapter } from './providers/gemini';
-
-export interface BenchmarkInput {
-  prompt: string;
-  workdir: string;
-  timeoutMs?: number;
-  /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */
-  providers: Array<'claude' | 'gpt' | 'gemini'>;
-  /** Optional per-provider model overrides. */
-  models?: Partial<Record<'claude' | 'gpt' | 'gemini', string>>;
-  /** If true, skip providers whose available() returns !ok. If false, include them with error. */
-  skipUnavailable?: boolean;
-}
-
-export interface BenchmarkEntry {
-  provider: string;
-  family: 'claude' | 'gpt' | 'gemini';
-  available: boolean;
-  unavailable_reason?: string;
-  result?: RunResult;
-  costUsd?: number;
-  /** Judge score 0-10 across dimensions. Populated separately by the judge step. */
-  qualityScore?: number;
-  qualityDetails?: Record<string, number>;
-}
-
-export interface BenchmarkReport {
-  prompt: string;
-  workdir: string;
-  startedAt: string;
-  durationMs: number;
-  entries: BenchmarkEntry[];
-}
-
-const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = {
-  claude: () => new ClaudeAdapter(),
-  gpt: () => new GptAdapter(),
-  gemini: () => new GeminiAdapter(),
-};
-
-export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkReport> {
-  const startedAtMs = Date.now();
-  const startedAt = new Date(startedAtMs).toISOString();
-  const timeoutMs = input.timeoutMs ?? 300_000;
-
-  const entries: BenchmarkEntry[] = [];
-  const runPromises: Array<Promise<void>> = [];
-
-  for (const name of input.providers) {
-    const factory = ADAPTERS[name];
-    if (!factory) {
-      entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` });
-      continue;
-    }
-    const adapter = factory();
-    const entry: BenchmarkEntry = { provider: adapter.name, family: adapter.family, available: true };
-    entries.push(entry);
-
-    runPromises.push((async () => {
-      const check = await adapter.available();
-      entry.available = check.ok;
-      if (!check.ok) {
-        entry.unavailable_reason = check.reason;
-        if (input.skipUnavailable) return;
-      }
-      const opts: RunOpts = {
-        prompt: input.prompt,
-        workdir: input.workdir,
-        timeoutMs,
-        model: input.models?.[name],
-      };
-      const res = await adapter.run(opts);
-      entry.result = res;
-      entry.costUsd = adapter.estimateCost(res.tokens, res.modelUsed);
-    })());
-  }
-
-  await Promise.allSettled(runPromises);
-
-  return {
-    prompt: input.prompt,
-    workdir: input.workdir,
-    startedAt,
-    durationMs: Date.now() - startedAtMs,
-    entries,
-  };
-}
-
-export function formatTable(report: BenchmarkReport): string {
-  const header = `Model                Latency   In→Out Tokens       Cost       Quality   Tool Calls   Notes`;
-  const sep = '-'.repeat(header.length);
-  const rows: string[] = [header, sep];
-  for (const e of report.entries) {
-    if (!e.available) {
-      rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
-      continue;
-    }
-    const r = e.result!;
-    if (r.error) {
-      rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
-      continue;
-    }
-    const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
-    rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
-  }
-  return rows.join('\n');
-}
-
-export function formatJson(report: BenchmarkReport): string {
-  return JSON.stringify(report, null, 2);
-}
-
-export function formatMarkdown(report: BenchmarkReport): string {
-  const lines: string[] = [
-    `# Benchmark report — ${report.startedAt}`,
-    '',
-    `**Prompt:** ${report.prompt.length > 200 ? report.prompt.slice(0, 200) + '…' : report.prompt}`,
-    `**Workdir:** \`${report.workdir}\``,
-    `**Total duration:** ${msToStr(report.durationMs)}`,
-    '',
-    '| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
-    '|-------|---------|-----------------|------|---------|-------|-------|',
-  ];
-  for (const e of report.entries) {
-    if (!e.available) {
-      lines.push(`| ${e.provider} | - | - | - | - | - | unavailable: ${e.unavailable_reason ?? 'unknown'} |`);
-      continue;
-    }
-    const r = e.result!;
-    if (r.error) {
-      lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
-      continue;
-    }
-    const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
-    lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
-  }
-  return lines.join('\n');
-}
-
-function pad(s: string, n: number): string {
-  return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
-}
-
-function msToStr(ms: number): string {
-  if (ms < 1000) return `${ms}ms`;
-  return `${(ms / 1000).toFixed(1)}s`;
-}
-
-function fmtCost(usd?: number): string {
-  if (usd === undefined) return '-';
-  if (usd < 0.01) return `$${usd.toFixed(4)}`;
-  return `$${usd.toFixed(2)}`;
-}

test/benchmark-cli.test.ts

@@ -1,7 +1,7 @@
 /**
  * gstack-model-benchmark CLI tests (offline).
  *
- * Covers CLI wiring that unit tests against benchmark-runner.ts can't see:
+ * Covers CLI wiring that unit tests can't see:
  *   - --dry-run auth/provider-list resolution
  *   - unknown provider WARN path
  *   - provider default (claude) when --models omitted
@@ -66,11 +66,22 @@ describe('gstack-model-benchmark --dry-run', () => {
     expect(r.stdout).toContain('providers:  claude');
   });
 
-  test('--timeout-ms and --workdir flags flow through to dry-run report', () => {
+  test('--timeout-ms flows through to dry-run report', () => {
-    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--workdir', '/tmp', '--dry-run']);
+    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--dry-run']);
     expect(r.status).toBe(0);
     expect(r.stdout).toContain('timeout_ms: 9999');
-    expect(r.stdout).toContain('workdir:    /tmp');
+  });
+
+  test('no prompt falls back to the corpus (not an error)', () => {
+    const r = run(['--models', 'claude', '--dry-run']);
+    expect(r.status).toBe(0);
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
+  });
+
+  test('upload is off by default (local only, nothing uploaded)', () => {
+    const r = run(['--prompt', 'hi', '--dry-run'], { env: { BRAINTRUST_API_KEY: '' } });
+    expect(r.status).toBe(0);
+    expect(r.stdout).toContain('local only');
   });
 
   test('--judge flag reported in dry-run output', () => {
@@ -196,9 +207,9 @@ describe('gstack-model-benchmark prompt resolution', () => {
     expect(r.stdout).toContain('treat-me-as-inline');
   });
 
-  test('missing prompt exits non-zero', () => {
+  test('no positional and no --prompt runs the corpus', () => {
     const r = run(['--dry-run']);
-    expect(r.status).not.toBe(0);
+    expect(r.status).toBe(0);
-    expect(r.stderr).toContain('specify a prompt');
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
   });
 });

test/benchmark-production-boundary.test.ts

@@ -42,12 +42,8 @@ test('production modules do not import from test directories', () => {
 
 test('former test-helper paths re-export the production benchmark API', async () => {
   const [
-    runner,
-    helperRunner,
     pricing,
     helperPricing,
-    judge,
-    helperJudge,
     claude,
     helperClaude,
     gpt,
@@ -55,12 +51,8 @@ test('former test-helper paths re-export the production benchmark API', async ()
     gemini,
     helperGemini,
   ] = await Promise.all([
-    import('../lib/model-benchmark/runner'),
-    import('./helpers/benchmark-runner'),
     import('../lib/model-benchmark/pricing'),
     import('./helpers/pricing'),
-    import('../lib/model-benchmark/judge'),
-    import('./helpers/benchmark-judge'),
     import('../lib/model-benchmark/providers/claude'),
     import('./helpers/providers/claude'),
     import('../lib/model-benchmark/providers/gpt'),
@@ -69,9 +61,7 @@ test('former test-helper paths re-export the production benchmark API', async ()
     import('./helpers/providers/gemini'),
   ]);
 
-  expect(helperRunner.runBenchmark).toBe(runner.runBenchmark);
   expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd);
-  expect(helperJudge.judgeEntries).toBe(judge.judgeEntries);
   expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
   expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
   expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);

test/benchmark-runner.test.ts

@@ -1,17 +1,12 @@
 /**
- * Unit tests for the benchmark runner.
+ * Unit tests for benchmark pricing + tool-compatibility helpers.
  *
- * Mocks adapters to verify:
+ * Orchestration, scoring, and reporting moved to Braintrust
- * - All adapters run in parallel (Promise.allSettled not serial)
+ * (lib/model-benchmark/braintrust-eval.ts); those are covered by
- * - Unavailable adapters are skipped or marked depending on flag
+ * model-benchmark-braintrust.test.ts and the live e2e suite.
- * - Per-adapter errors don't abort the batch
- * - Output formatters (table, json, markdown) produce non-empty strings
- *
- * Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those.
  */
 
 import { test, expect } from 'bun:test';
-import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from '../lib/model-benchmark/runner';
 import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing';
 import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
 
@@ -58,80 +53,3 @@ test('TOOL_COMPATIBILITY is populated for all three families', () => {
   expect(TOOL_COMPATIBILITY.gpt).toBeDefined();
   expect(TOOL_COMPATIBILITY.gemini).toBeDefined();
 });
-
-test('formatTable handles a report with mixed success/error/unavailable entries', () => {
-  const report: BenchmarkReport = {
-    prompt: 'test prompt',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 1500,
-    entries: [
-      {
-        provider: 'claude',
-        family: 'claude',
-        available: true,
-        result: {
-          output: 'ok',
-          tokens: { input: 100, output: 200 },
-          durationMs: 800,
-          toolCalls: 3,
-          modelUsed: 'claude-opus-4-7',
-        },
-        costUsd: 0.0165,
-        qualityScore: 9.2,
-      },
-      {
-        provider: 'gpt',
-        family: 'gpt',
-        available: true,
-        result: {
-          output: '',
-          tokens: { input: 0, output: 0 },
-          durationMs: 200,
-          toolCalls: 0,
-          modelUsed: 'gpt-5.4',
-          error: { code: 'auth', reason: 'codex login required' },
-        },
-      },
-      {
-        provider: 'gemini',
-        family: 'gemini',
-        available: false,
-        unavailable_reason: 'gemini CLI not on PATH',
-      },
-    ],
-  };
-
-  const table = formatTable(report);
-  expect(table).toContain('claude-opus-4-7');
-  expect(table).toContain('ERROR auth');
-  expect(table).toContain('unavailable');
-  expect(table).toContain('9.2/10');
-});
-
-test('formatJson produces parseable JSON', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const json = formatJson(report);
-  const parsed = JSON.parse(json);
-  expect(parsed.prompt).toBe('x');
-  expect(parsed.entries).toEqual([]);
-});
-
-test('formatMarkdown produces a table header', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const md = formatMarkdown(report);
-  expect(md).toContain('# Benchmark report');
-  expect(md).toContain('| Model | Latency |');
-});

test/helpers/benchmark-judge.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/judge';

test/helpers/benchmark-runner.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/runner';

test/skill-e2e-benchmark-providers.test.ts

@@ -14,7 +14,7 @@
  *   - Parallel execution via Promise.allSettled — slow provider doesn't block fast
  *
  * NOT covered here (would need dedicated test files):
- *   - Quality judge integration (benchmark-judge.ts, adds ~$0.05/run)
+ *   - Quality judge integration (autoevals ClosedQA, opt-in)
  *   - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0`
  */
 
@@ -22,7 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
 import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
 import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';
-import { runBenchmark } from '../lib/model-benchmark/runner';
+import { runProviderBenchmark } from '../lib/model-benchmark/braintrust-eval';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -163,30 +163,23 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     expect(result.durationMs).toBeGreaterThan(0);
   }, 30_000);
 
-  test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => {
+  test('runProviderBenchmark: an unauthed/failing provider returns a result, never throws', async () => {
-    // Use the full runner with all three providers — whichever are unauthed should
+    // Braintrust owns orchestration now. The property we care about: a provider
-    // return entries with available=false and not crash the batch.
+    // that's unavailable or errors comes back as a ProviderBenchmark (score null,
-    const report = await runBenchmark({
+    // ops carrying the error) instead of throwing and aborting the batch.
-      prompt: PROMPT,
+    const cases = [{ id: 'smoke', input: PROMPT, required: ['ok'] }];
-      workdir,
+    const results = await Promise.all(
-      providers: ['claude', 'gpt', 'gemini'],
+      (['claude', 'gpt', 'gemini'] as const).map(p => runProviderBenchmark(p, cases, { timeoutMs: 120_000 })),
-      timeoutMs: 120_000,
+    );
-      skipUnavailable: false,
+    expect(results).toHaveLength(3);
-    });
+    for (const r of results) {
-    expect(report.entries).toHaveLength(3);
+      expect(['claude', 'gpt', 'gemini']).toContain(r.provider);
-    for (const e of report.entries) {
+      expect(r.score === null || (typeof r.score === 'number' && r.score >= 0 && r.score <= 1)).toBe(true);
-      expect(['claude', 'gpt', 'gemini']).toContain(e.family);
+      expect(Array.isArray(r.ops)).toBe(true);
-      if (e.available) {
-        expect(e.result).toBeDefined();
-      } else {
-        expect(typeof e.unavailable_reason).toBe('string');
-      }
     }
-    // At least one available provider should have produced a non-error result in a healthy CI env.
+    const hadSuccess = results.some(r => typeof r.score === 'number' && r.ops.some(o => !o.error));
-    const hadSuccess = report.entries.some(e => e.available && e.result && !e.result.error);
-    // We don't hard-assert this: if NO providers are authed, skip silently.
     if (!hadSuccess) {
-      process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
+      process.stderr.write('\nbenchmark live: no provider produced a clean result (no auth?)\n');
     }
   }, 300_000);
 });