CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-21 20:30:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: back model benchmark with Braintrust, drop in-house scoring

Browse Source 
Braintrust now owns benchmark scoring, experiments, comparison, and
reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim)
plus operational metrics the CLIs report. runProviderBenchmark wraps each
adapter as a Braintrust Eval task; a deterministic required-terms scorer
replaces the in-house evaluation logic and the optional autoevals ClosedQA
judge replaces judge.ts.

Runs local by default under bun: with no BRAINTRUST_API_KEY it sets
noSendLogs and ships nothing; setting the key opts into the cloud dashboard.
An empty output with zero tokens is thrown so a silent auth/CLI failure
can't masquerade as a 0.0 score.

Deletes runner.ts and judge.ts (and their test-helper re-export shims);
rewires bin/gstack-model-benchmark and adapts the benchmark tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sinabina
2026-07-21 14:12:10 -07:00
co-authored by Claude Opus 4.8
parent e71e7f7f41
commit 36f972f2e4
12 changed files with 321 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-benchmark-providers.test.ts
+17 -24
View File
@@ -14,7 +14,7 @@
* - Parallel execution via Promise.allSettled — slow provider doesn't block fast
*
* NOT covered here (would need dedicated test files):
* - Quality judge integration (benchmark-judge.ts, adds ~$0.05/run)
* - Quality judge integration (autoevals ClosedQA, opt-in)
* - Multi-turn tool-using prompts — our single-turn smoke skips `toolCalls > 0`
*/
@@ -22,7 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';
import { runBenchmark } from '../lib/model-benchmark/runner';
import { runProviderBenchmark } from '../lib/model-benchmark/braintrust-eval';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -163,30 +163,23 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.durationMs).toBeGreaterThan(0);
}, 30_000);
test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => {
// Use the full runner with all three providers — whichever are unauthed should
// return entries with available=false and not crash the batch.
const report = await runBenchmark({
prompt: PROMPT,
workdir,
providers: ['claude', 'gpt', 'gemini'],
timeoutMs: 120_000,
skipUnavailable: false,
});
expect(report.entries).toHaveLength(3);
for (const e of report.entries) {
expect(['claude', 'gpt', 'gemini']).toContain(e.family);
if (e.available) {
expect(e.result).toBeDefined();
} else {
expect(typeof e.unavailable_reason).toBe('string');
}
test('runProviderBenchmark: an unauthed/failing provider returns a result, never throws', async () => {
// Braintrust owns orchestration now. The property we care about: a provider
// that's unavailable or errors comes back as a ProviderBenchmark (score null,
// ops carrying the error) instead of throwing and aborting the batch.
const cases = [{ id: 'smoke', input: PROMPT, required: ['ok'] }];
const results = await Promise.all(
(['claude', 'gpt', 'gemini'] as const).map(p => runProviderBenchmark(p, cases, { timeoutMs: 120_000 })),
);
expect(results).toHaveLength(3);
for (const r of results) {
expect(['claude', 'gpt', 'gemini']).toContain(r.provider);
expect(r.score === null || (typeof r.score === 'number' && r.score >= 0 && r.score <= 1)).toBe(true);
expect(Array.isArray(r.ops)).toBe(true);
}
// At least one available provider should have produced a non-error result in a healthy CI env.
const hadSuccess = report.entries.some(e => e.available && e.result && !e.result.error);
// We don't hard-assert this: if NO providers are authed, skip silently.
const hadSuccess = results.some(r => typeof r.score === 'number' && r.ops.some(o => !o.error));
if (!hadSuccess) {
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
process.stderr.write('\nbenchmark live: no provider produced a clean result (no auth?)\n');
}
}, 300_000);
});