Back model benchmark with Braintrust, drop in-house scoring (#13)

# Conflicts:
#	bun.lock
#	package.json

test/benchmark-cli.test.ts

@@ -1,7 +1,7 @@
 /**
  * gstack-model-benchmark CLI tests (offline).
  *
- * Covers CLI wiring that unit tests against benchmark-runner.ts can't see:
+ * Covers CLI wiring that unit tests can't see:
  *   - --dry-run auth/provider-list resolution
  *   - unknown provider WARN path
  *   - provider default (claude) when --models omitted
@@ -66,11 +66,22 @@ describe('gstack-model-benchmark --dry-run', () => {
     expect(r.stdout).toContain('providers:  claude');
   });
 
-  test('--timeout-ms and --workdir flags flow through to dry-run report', () => {
-    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--workdir', '/tmp', '--dry-run']);
+  test('--timeout-ms flows through to dry-run report', () => {
+    const r = run(['--prompt', 'hi', '--timeout-ms', '9999', '--dry-run']);
     expect(r.status).toBe(0);
     expect(r.stdout).toContain('timeout_ms: 9999');
-    expect(r.stdout).toContain('workdir:    /tmp');
+  });
+
+  test('no prompt falls back to the corpus (not an error)', () => {
+    const r = run(['--models', 'claude', '--dry-run']);
+    expect(r.status).toBe(0);
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
+  });
+
+  test('upload is off by default (local only, nothing uploaded)', () => {
+    const r = run(['--prompt', 'hi', '--dry-run'], { env: { BRAINTRUST_API_KEY: '' } });
+    expect(r.status).toBe(0);
+    expect(r.stdout).toContain('local only');
   });
 
   test('--judge flag reported in dry-run output', () => {
@@ -196,9 +207,9 @@ describe('gstack-model-benchmark prompt resolution', () => {
     expect(r.stdout).toContain('treat-me-as-inline');
   });
 
-  test('missing prompt exits non-zero', () => {
+  test('no positional and no --prompt runs the corpus', () => {
     const r = run(['--dry-run']);
-    expect(r.status).not.toBe(0);
-    expect(r.stderr).toContain('specify a prompt');
+    expect(r.status).toBe(0);
+    expect(r.stdout).toMatch(/corpus:\s+\d+ case/);
   });
 });

test/benchmark-production-boundary.test.ts

@@ -42,12 +42,6 @@ test('production modules do not import from test directories', () => {
 
 test('former test-helper paths re-export the production benchmark API', async () => {
   const [
-    runner,
-    helperRunner,
-    pricing,
-    helperPricing,
-    judge,
-    helperJudge,
     claude,
     helperClaude,
     gpt,
@@ -55,12 +49,6 @@ test('former test-helper paths re-export the production benchmark API', async ()
     gemini,
     helperGemini,
   ] = await Promise.all([
-    import('../lib/model-benchmark/runner'),
-    import('./helpers/benchmark-runner'),
-    import('../lib/model-benchmark/pricing'),
-    import('./helpers/pricing'),
-    import('../lib/model-benchmark/judge'),
-    import('./helpers/benchmark-judge'),
     import('../lib/model-benchmark/providers/claude'),
     import('./helpers/providers/claude'),
     import('../lib/model-benchmark/providers/gpt'),
@@ -69,9 +57,6 @@ test('former test-helper paths re-export the production benchmark API', async ()
     import('./helpers/providers/gemini'),
   ]);
 
-  expect(helperRunner.runBenchmark).toBe(runner.runBenchmark);
-  expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd);
-  expect(helperJudge.judgeEntries).toBe(judge.judgeEntries);
   expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
   expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
   expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);

test/benchmark-runner.test.ts

@@ -1,49 +1,14 @@
 /**
- * Unit tests for the benchmark runner.
+ * Unit tests for benchmark tool-compatibility helpers.
  *
- * Mocks adapters to verify:
- * - All adapters run in parallel (Promise.allSettled not serial)
- * - Unavailable adapters are skipped or marked depending on flag
- * - Per-adapter errors don't abort the batch
- * - Output formatters (table, json, markdown) produce non-empty strings
- *
- * Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those.
+ * Orchestration and scoring moved to Braintrust (lib/model-benchmark/braintrust-eval.ts);
+ * per-provider token/cost tracking was removed with the JSON-schema parsers.
+ * Provider capability coverage is what's left to pin here.
  */
 
 import { test, expect } from 'bun:test';
-import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from '../lib/model-benchmark/runner';
-import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing';
 import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
 
-test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
-  const cost = estimateCostUsd({ input: 1000, output: 500 }, 'unknown-model-7b');
-  expect(cost).toBe(0);
-});
-
-test('estimateCostUsd computes correctly for known Claude model', () => {
-  // claude-opus-4-7: $15/MTok input, $75/MTok output
-  // 1M input + 0.5M output = $15 + $37.50 = $52.50
-  const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7');
-  expect(cost).toBeCloseTo(52.50, 2);
-});
-
-test('estimateCostUsd applies cached input discount alongside uncached input', () => {
-  // tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%.
-  // 0 uncached input, 1M cached → 10% of 15 = $1.50
-  const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7');
-  expect(cost1).toBeCloseTo(1.50, 2);
-  // 500K uncached input + 500K cached → $7.50 + $0.75 = $8.25
-  const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7');
-  expect(cost2).toBeCloseTo(8.25, 2);
-});
-
-test('PRICING table covers the key model families', () => {
-  expect(PRICING['claude-opus-4-7']).toBeDefined();
-  expect(PRICING['claude-sonnet-4-6']).toBeDefined();
-  expect(PRICING['gpt-5.4']).toBeDefined();
-  expect(PRICING['gemini-2.5-pro']).toBeDefined();
-});
-
 test('missingTools reports unsupported tools per provider', () => {
   // GPT/Codex doesn't expose Edit, Glob, Grep
   expect(missingTools('gpt', ['Edit', 'Glob', 'Grep'])).toEqual(['Edit', 'Glob', 'Grep']);
@@ -58,80 +23,3 @@ test('TOOL_COMPATIBILITY is populated for all three families', () => {
   expect(TOOL_COMPATIBILITY.gpt).toBeDefined();
   expect(TOOL_COMPATIBILITY.gemini).toBeDefined();
 });
-
-test('formatTable handles a report with mixed success/error/unavailable entries', () => {
-  const report: BenchmarkReport = {
-    prompt: 'test prompt',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 1500,
-    entries: [
-      {
-        provider: 'claude',
-        family: 'claude',
-        available: true,
-        result: {
-          output: 'ok',
-          tokens: { input: 100, output: 200 },
-          durationMs: 800,
-          toolCalls: 3,
-          modelUsed: 'claude-opus-4-7',
-        },
-        costUsd: 0.0165,
-        qualityScore: 9.2,
-      },
-      {
-        provider: 'gpt',
-        family: 'gpt',
-        available: true,
-        result: {
-          output: '',
-          tokens: { input: 0, output: 0 },
-          durationMs: 200,
-          toolCalls: 0,
-          modelUsed: 'gpt-5.4',
-          error: { code: 'auth', reason: 'codex login required' },
-        },
-      },
-      {
-        provider: 'gemini',
-        family: 'gemini',
-        available: false,
-        unavailable_reason: 'gemini CLI not on PATH',
-      },
-    ],
-  };
-
-  const table = formatTable(report);
-  expect(table).toContain('claude-opus-4-7');
-  expect(table).toContain('ERROR auth');
-  expect(table).toContain('unavailable');
-  expect(table).toContain('9.2/10');
-});
-
-test('formatJson produces parseable JSON', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const json = formatJson(report);
-  const parsed = JSON.parse(json);
-  expect(parsed.prompt).toBe('x');
-  expect(parsed.entries).toEqual([]);
-});
-
-test('formatMarkdown produces a table header', () => {
-  const report: BenchmarkReport = {
-    prompt: 'x',
-    workdir: '/tmp',
-    startedAt: '2026-04-16T20:00:00Z',
-    durationMs: 100,
-    entries: [],
-  };
-  const md = formatMarkdown(report);
-  expect(md).toContain('# Benchmark report');
-  expect(md).toContain('| Model | Latency |');
-});

test/helpers/benchmark-judge.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/judge';

test/helpers/benchmark-runner.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/runner';

test/helpers/pricing.ts

@@ -1,2 +0,0 @@
-// Compatibility export for tests and downstream tooling that used the former helper path.
-export * from '../../lib/model-benchmark/pricing';

test/skill-e2e-benchmark-providers.test.ts

@@ -7,22 +7,19 @@
  * to keep cost near $0.001/provider/run.
  *
  * What this catches that unit tests don't:
- *   - CLI output-format drift (the #1 silent breakage path)
- *   - Token parsing from real provider responses
+ *   - CLI invocation drift (a flag rename or trust-prompt change breaking a run)
  *   - Auth-failure vs timeout vs rate-limit error code routing
- *   - Cost estimation on real token counts
- *   - Parallel execution via Promise.allSettled — slow provider doesn't block fast
+ *   - The adapter terminates without throwing and returns plain-text output
  *
  * NOT covered here (would need dedicated test files):
- *   - Quality judge integration (benchmark-judge.ts, adds ~$0.05/run)
- *   - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0`
+ *   - Quality judge integration (autoevals ClosedQA, opt-in)
  */
 
 import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
 import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
 import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';
-import { runBenchmark } from '../lib/model-benchmark/runner';
+import { runProviderBenchmark } from '../lib/model-benchmark/braintrust-eval';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -91,13 +88,9 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
       throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`);
     }
     expect(result.output.toLowerCase()).toContain('ok');
-    expect(result.tokens.input).toBeGreaterThan(0);
-    expect(result.tokens.output).toBeGreaterThan(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
     expect(result.modelUsed.length).toBeGreaterThan(0);
-    const cost = claude.estimateCost(result.tokens, result.modelUsed);
-    expect(cost).toBeGreaterThan(0);
   }, 150_000);
 
   test('gpt: trivial prompt produces parseable output', async () => {
@@ -111,12 +104,8 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
       throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`);
     }
     expect(result.output.toLowerCase()).toContain('ok');
-    expect(result.tokens.input).toBeGreaterThan(0);
-    expect(result.tokens.output).toBeGreaterThan(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
-    const cost = gpt.estimateCost(result.tokens, result.modelUsed);
-    expect(cost).toBeGreaterThan(0);
   }, 150_000);
 
   test('gemini: trivial prompt produces parseable output', async () => {
@@ -129,17 +118,10 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     if (result.error) {
       throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`);
     }
-    // Gemini CLI occasionally returns empty output even on successful runs
-    // (model returned content the CLI parser missed, intermittent stream issues).
-    // We assert the adapter ran end-to-end without erroring and reports a non-
-    // empty token count instead of grepping the literal "ok" — that string
-    // assertion was too brittle for a smoke that's really about "did the
-    // adapter wire up and the run terminate successfully?"
+    // Gemini CLI can return empty output on otherwise-successful runs in some
+    // environments. This smoke is about "did the adapter wire up and terminate
+    // without throwing" — assert the shape, not the content.
     expect(typeof result.output).toBe('string');
-    // Gemini CLI sometimes returns 0 tokens in the result event (older responses);
-    // assert non-negative instead of strictly positive.
-    expect(result.tokens.input).toBeGreaterThanOrEqual(0);
-    expect(result.tokens.output).toBeGreaterThanOrEqual(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
   }, 150_000);
@@ -163,30 +145,23 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     expect(result.durationMs).toBeGreaterThan(0);
   }, 30_000);
 
-  test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => {
-    // Use the full runner with all three providers — whichever are unauthed should
-    // return entries with available=false and not crash the batch.
-    const report = await runBenchmark({
-      prompt: PROMPT,
-      workdir,
-      providers: ['claude', 'gpt', 'gemini'],
-      timeoutMs: 120_000,
-      skipUnavailable: false,
-    });
-    expect(report.entries).toHaveLength(3);
-    for (const e of report.entries) {
-      expect(['claude', 'gpt', 'gemini']).toContain(e.family);
-      if (e.available) {
-        expect(e.result).toBeDefined();
-      } else {
-        expect(typeof e.unavailable_reason).toBe('string');
-      }
+  test('runProviderBenchmark: an unauthed/failing provider returns a result, never throws', async () => {
+    // Braintrust owns orchestration now. The property we care about: a provider
+    // that's unavailable or errors comes back as a ProviderBenchmark (score null,
+    // ops carrying the error) instead of throwing and aborting the batch.
+    const cases = [{ id: 'smoke', input: PROMPT, required: ['ok'] }];
+    const results = await Promise.all(
+      (['claude', 'gpt', 'gemini'] as const).map(p => runProviderBenchmark(p, cases, { timeoutMs: 120_000 })),
+    );
+    expect(results).toHaveLength(3);
+    for (const r of results) {
+      expect(['claude', 'gpt', 'gemini']).toContain(r.provider);
+      expect(r.score === null || (typeof r.score === 'number' && r.score >= 0 && r.score <= 1)).toBe(true);
+      expect(Array.isArray(r.ops)).toBe(true);
     }
-    // At least one available provider should have produced a non-error result in a healthy CI env.
-    const hadSuccess = report.entries.some(e => e.available && e.result && !e.result.error);
-    // We don't hard-assert this: if NO providers are authed, skip silently.
+    const hadSuccess = results.some(r => typeof r.score === 'number' && r.ops.some(o => !o.error));
     if (!hadSuccess) {
-      process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
+      process.stderr.write('\nbenchmark live: no provider produced a clean result (no auth?)\n');
     }
   }, 300_000);
 });