mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 12:20:48 +02:00
feat: back model benchmark with Braintrust, drop in-house scoring
Braintrust now owns benchmark scoring, experiments, comparison, and reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim) plus operational metrics the CLIs report. runProviderBenchmark wraps each adapter as a Braintrust Eval task; a deterministic required-terms scorer replaces the in-house evaluation logic and the optional autoevals ClosedQA judge replaces judge.ts. Runs local by default under bun: with no BRAINTRUST_API_KEY it sets noSendLogs and ships nothing; setting the key opts into the cloud dashboard. An empty output with zero tokens is thrown so a silent auth/CLI failure can't masquerade as a 0.0 score. Deletes runner.ts and judge.ts (and their test-helper re-export shims); rewires bin/gstack-model-benchmark and adapts the benchmark tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
e71e7f7f41
commit
36f972f2e4
@@ -1,17 +1,12 @@
|
||||
/**
|
||||
* Unit tests for the benchmark runner.
|
||||
* Unit tests for benchmark pricing + tool-compatibility helpers.
|
||||
*
|
||||
* Mocks adapters to verify:
|
||||
* - All adapters run in parallel (Promise.allSettled not serial)
|
||||
* - Unavailable adapters are skipped or marked depending on flag
|
||||
* - Per-adapter errors don't abort the batch
|
||||
* - Output formatters (table, json, markdown) produce non-empty strings
|
||||
*
|
||||
* Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those.
|
||||
* Orchestration, scoring, and reporting moved to Braintrust
|
||||
* (lib/model-benchmark/braintrust-eval.ts); those are covered by
|
||||
* model-benchmark-braintrust.test.ts and the live e2e suite.
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from '../lib/model-benchmark/runner';
|
||||
import { estimateCostUsd, PRICING } from '../lib/model-benchmark/pricing';
|
||||
import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
|
||||
|
||||
@@ -58,80 +53,3 @@ test('TOOL_COMPATIBILITY is populated for all three families', () => {
|
||||
expect(TOOL_COMPATIBILITY.gpt).toBeDefined();
|
||||
expect(TOOL_COMPATIBILITY.gemini).toBeDefined();
|
||||
});
|
||||
|
||||
test('formatTable handles a report with mixed success/error/unavailable entries', () => {
|
||||
const report: BenchmarkReport = {
|
||||
prompt: 'test prompt',
|
||||
workdir: '/tmp',
|
||||
startedAt: '2026-04-16T20:00:00Z',
|
||||
durationMs: 1500,
|
||||
entries: [
|
||||
{
|
||||
provider: 'claude',
|
||||
family: 'claude',
|
||||
available: true,
|
||||
result: {
|
||||
output: 'ok',
|
||||
tokens: { input: 100, output: 200 },
|
||||
durationMs: 800,
|
||||
toolCalls: 3,
|
||||
modelUsed: 'claude-opus-4-7',
|
||||
},
|
||||
costUsd: 0.0165,
|
||||
qualityScore: 9.2,
|
||||
},
|
||||
{
|
||||
provider: 'gpt',
|
||||
family: 'gpt',
|
||||
available: true,
|
||||
result: {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs: 200,
|
||||
toolCalls: 0,
|
||||
modelUsed: 'gpt-5.4',
|
||||
error: { code: 'auth', reason: 'codex login required' },
|
||||
},
|
||||
},
|
||||
{
|
||||
provider: 'gemini',
|
||||
family: 'gemini',
|
||||
available: false,
|
||||
unavailable_reason: 'gemini CLI not on PATH',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const table = formatTable(report);
|
||||
expect(table).toContain('claude-opus-4-7');
|
||||
expect(table).toContain('ERROR auth');
|
||||
expect(table).toContain('unavailable');
|
||||
expect(table).toContain('9.2/10');
|
||||
});
|
||||
|
||||
test('formatJson produces parseable JSON', () => {
|
||||
const report: BenchmarkReport = {
|
||||
prompt: 'x',
|
||||
workdir: '/tmp',
|
||||
startedAt: '2026-04-16T20:00:00Z',
|
||||
durationMs: 100,
|
||||
entries: [],
|
||||
};
|
||||
const json = formatJson(report);
|
||||
const parsed = JSON.parse(json);
|
||||
expect(parsed.prompt).toBe('x');
|
||||
expect(parsed.entries).toEqual([]);
|
||||
});
|
||||
|
||||
test('formatMarkdown produces a table header', () => {
|
||||
const report: BenchmarkReport = {
|
||||
prompt: 'x',
|
||||
workdir: '/tmp',
|
||||
startedAt: '2026-04-16T20:00:00Z',
|
||||
durationMs: 100,
|
||||
entries: [],
|
||||
};
|
||||
const md = formatMarkdown(report);
|
||||
expect(md).toContain('# Benchmark report');
|
||||
expect(md).toContain('| Model | Latency |');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user