feat: back model benchmark with Braintrust, drop in-house scoring

Braintrust now owns benchmark scoring, experiments, comparison, and
reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim)
plus operational metrics the CLIs report. runProviderBenchmark wraps each
adapter as a Braintrust Eval task; a deterministic required-terms scorer
replaces the in-house evaluation logic and the optional autoevals ClosedQA
judge replaces judge.ts.

Runs local by default under bun: with no BRAINTRUST_API_KEY it sets
noSendLogs and ships nothing; setting the key opts into the cloud dashboard.
An empty output with zero tokens is thrown so a silent auth/CLI failure
can't masquerade as a 0.0 score.

Deletes runner.ts and judge.ts (and their test-helper re-export shims);
rewires bin/gstack-model-benchmark and adapts the benchmark tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

test/benchmark-cli.test.ts

@@ -1,7 +1,7 @@
 /**
  * gstack-model-benchmark CLI tests (offline).
  *
- * Covers CLI wiring that unit tests against benchmark-runner.ts can't see:
+ * Covers CLI wiring that unit tests can't see:
  *   - --dry-run auth/provider-list resolution
  *   - unknown provider WARN path
  *   - provider default (claude) when --models omitted
@@ -66,11 +66,22 @@ describe('gstack-model-benchmark --dry-run', () => {
     expect(r.stdout).toContain('providers:  claude');
   });
 
-  test('--timeout-ms and --workdir flags flow through to dry-run report', () => {
-    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--workdir', '/tmp', '--dry-run']);
+  test('--timeout-ms flows through to dry-run report', () => {
+    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--dry-run']);
     expect(r.status).toBe(0);
     expect(r.stdout).toContain('timeout_ms: 9999');
-    expect(r.stdout).toContain('workdir:    /tmp');
+  });
+
+  test('no prompt falls back to the corpus (not an error)', () => {
+    const r = run(['--models', 'claude', '--dry-run']);
+    expect(r.status).toBe(0);
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
+  });
+
+  test('upload is off by default (local only, nothing uploaded)', () => {
+    const r = run(['--prompt', 'hi', '--dry-run'], { env: { BRAINTRUST_API_KEY: '' } });
+    expect(r.status).toBe(0);
+    expect(r.stdout).toContain('local only');
   });
 
   test('--judge flag reported in dry-run output', () => {
@@ -196,9 +207,9 @@ describe('gstack-model-benchmark prompt resolution', () => {
     expect(r.stdout).toContain('treat-me-as-inline');
   });
 
-  test('missing prompt exits non-zero', () => {
+  test('no positional and no --prompt runs the corpus', () => {
     const r = run(['--dry-run']);
-    expect(r.status).not.toBe(0);
-    expect(r.stderr).toContain('specify a prompt');
+    expect(r.status).toBe(0);
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
   });
 });

test/benchmark-production-boundary.test.ts

@@ -42,12 +42,8 @@ test('production modules do not import from test directories', () => {
 
 test('former test-helper paths re-export the production benchmark API', async () => {
   const [
-    runner,
-    helperRunner,
     pricing,
     helperPricing,
-    judge,
-    helperJudge,
     claude,
     helperClaude,
     gpt,
@@ -55,12 +51,8 @@ test('former test-helper paths re-export the production benchmark API', async ()
     gemini,
     helperGemini,
   ] = await Promise.all([
-    import('../lib/model-benchmark/runner'),
-    import('./helpers/benchmark-runner'),
     import('../lib/model-benchmark/pricing'),
     import('./helpers/pricing'),
-    import('../lib/model-benchmark/judge'),
-    import('./helpers/benchmark-judge'),
     import('../lib/model-benchmark/providers/claude'),
     import('./helpers/providers/claude'),
     import('../lib/model-benchmark/providers/gpt'),
@@ -69,9 +61,7 @@ test('former test-helper paths re-export the production benchmark API', async ()
     import('./helpers/providers/gemini'),
   ]);
 
-  expect(helperRunner.runBenchmark).toBe(runner.runBenchmark);
   expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd);
-  expect(helperJudge.judgeEntries).toBe(judge.judgeEntries);
   expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
   expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
   expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);

test/benchmark-runner.test.ts

@@ -1,17 +1,12 @@
 /**
- * Unit tests for the benchmark runner.
+ * Unit tests for benchmark pricing + tool-compatibility helpers.
  *
- * Mocks adapters to verify:
- * - All adapters run in parallel (Promise.allSettled not serial)
- * - Unavailable adapters are skipped or marked depending on flag
- * - Per-adapter errors don't abort the batch
- * - Output formatters (table, json, markdown) produce non-empty strings
- *
- * Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those.
+ * Orchestration, scoring, and reporting moved to Braintrust
+ * (lib/model-benchmark/braintrust-eval.ts); those are covered by
+ * model-benchmark-braintrust.test.ts and the live e2e suite.
  */
 
 import { test, expect } from 'bun:test';
-import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from '../lib/model-benchmark/runner';
 import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing';
 import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
 
@@ -58,80 +53,3 @@ test('TOOL_COMPATIBILITY is populated for all three families', () => {
   expect(TOOL_COMPATIBILITY.gpt).toBeDefined();
   expect(TOOL_COMPATIBILITY.gemini).toBeDefined();
 });
-
-test('formatTable handles a report with mixed success/error/unavailable entries', () => {
-  const report: BenchmarkReport = {
-    prompt: 'test prompt',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 1500,
-    entries: [
-      {
-        provider: 'claude',
-        family: 'claude',
-        available: true,
-        result: {
-          output: 'ok',
-          tokens: { input: 100, output: 200 },
-          durationMs: 800,
-          toolCalls: 3,
-          modelUsed: 'claude-opus-4-7',
-        },
-        costUsd: 0.0165,
-        qualityScore: 9.2,
-      },
-      {
-        provider: 'gpt',
-        family: 'gpt',
-        available: true,
-        result: {
-          output: '',
-          tokens: { input: 0, output: 0 },
-          durationMs: 200,
-          toolCalls: 0,
-          modelUsed: 'gpt-5.4',
-          error: { code: 'auth', reason: 'codex login required' },
-        },
-      },
-      {
-        provider: 'gemini',
-        family: 'gemini',
-        available: false,
-        unavailable_reason: 'gemini CLI not on PATH',
-      },
-    ],
-  };
-
-  const table = formatTable(report);
-  expect(table).toContain('claude-opus-4-7');
-  expect(table).toContain('ERROR auth');
-  expect(table).toContain('unavailable');
-  expect(table).toContain('9.2/10');
-});
-
-test('formatJson produces parseable JSON', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const json = formatJson(report);
-  const parsed = JSON.parse(json);
-  expect(parsed.prompt).toBe('x');
-  expect(parsed.entries).toEqual([]);
-});
-
-test('formatMarkdown produces a table header', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const md = formatMarkdown(report);
-  expect(md).toContain('# Benchmark report');
-  expect(md).toContain('| Model | Latency |');
-});

test/helpers/benchmark-judge.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/judge';

test/helpers/benchmark-runner.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/runner';

test/skill-e2e-benchmark-providers.test.ts

@@ -14,7 +14,7 @@
  *   - Parallel execution via Promise.allSettled — slow provider doesn't block fast
  *
  * NOT covered here (would need dedicated test files):
- *   - Quality judge integration (benchmark-judge.ts, adds ~$0.05/run)
+ *   - Quality judge integration (autoevals ClosedQA, opt-in)
  *   - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0`
  */
 
@@ -22,7 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
 import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
 import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';
-import { runBenchmark } from '../lib/model-benchmark/runner';
+import { runProviderBenchmark } from '../lib/model-benchmark/braintrust-eval';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -163,30 +163,23 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     expect(result.durationMs).toBeGreaterThan(0);
   }, 30_000);
 
-  test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => {
-    // Use the full runner with all three providers — whichever are unauthed should
-    // return entries with available=false and not crash the batch.
-    const report = await runBenchmark({
-      prompt: PROMPT,
-      workdir,
-      providers: ['claude', 'gpt', 'gemini'],
-      timeoutMs: 120_000,
-      skipUnavailable: false,
-    });
-    expect(report.entries).toHaveLength(3);
-    for (const e of report.entries) {
-      expect(['claude', 'gpt', 'gemini']).toContain(e.family);
-      if (e.available) {
-        expect(e.result).toBeDefined();
-      } else {
-        expect(typeof e.unavailable_reason).toBe('string');
-      }
+  test('runProviderBenchmark: an unauthed/failing provider returns a result, never throws', async () => {
+    // Braintrust owns orchestration now. The property we care about: a provider
+    // that's unavailable or errors comes back as a ProviderBenchmark (score null,
+    // ops carrying the error) instead of throwing and aborting the batch.
+    const cases = [{ id: 'smoke', input: PROMPT, required: ['ok'] }];
+    const results = await Promise.all(
+      (['claude', 'gpt', 'gemini'] as const).map(p => runProviderBenchmark(p, cases, { timeoutMs: 120_000 })),
+    );
+    expect(results).toHaveLength(3);
+    for (const r of results) {
+      expect(['claude', 'gpt', 'gemini']).toContain(r.provider);
+      expect(r.score === null || (typeof r.score === 'number' && r.score >= 0 && r.score <= 1)).toBe(true);
+      expect(Array.isArray(r.ops)).toBe(true);
     }
-    // At least one available provider should have produced a non-error result in a healthy CI env.
-    const hadSuccess = report.entries.some(e => e.available && e.result && !e.result.error);
-    // We don't hard-assert this: if NO providers are authed, skip silently.
+    const hadSuccess = results.some(r => typeof r.score === 'number' && r.ops.some(o => !o.error));
     if (!hadSuccess) {
-      process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
+      process.stderr.write('\nbenchmark live: no provider produced a clean result (no auth?)\n');
     }
   }, 300_000);
 });