mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 12:20:48 +02:00
feat: back model benchmark with Braintrust, drop in-house scoring
Braintrust now owns benchmark scoring, experiments, comparison, and reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim) plus operational metrics the CLIs report. runProviderBenchmark wraps each adapter as a Braintrust Eval task; a deterministic required-terms scorer replaces the in-house evaluation logic and the optional autoevals ClosedQA judge replaces judge.ts. Runs local by default under bun: with no BRAINTRUST_API_KEY it sets noSendLogs and ships nothing; setting the key opts into the cloud dashboard. An empty output with zero tokens is thrown so a silent auth/CLI failure can't masquerade as a 0.0 score. Deletes runner.ts and judge.ts (and their test-helper re-export shims); rewires bin/gstack-model-benchmark and adapts the benchmark tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sinabina
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
e71e7f7f41
commit
36f972f2e4
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* Braintrust-backed benchmark core.
|
||||
*
|
||||
* Braintrust owns scoring, experiment tracking, comparison, and reporting. GStack
|
||||
* keeps only the unavoidable CLI-agent shims (providers/*.ts) as the eval `task`,
|
||||
* plus operational metrics (latency/tokens/cost) the CLIs report and Braintrust
|
||||
* doesn't measure for us.
|
||||
*
|
||||
* Local by default: with no BRAINTRUST_API_KEY, runs with `noSendLogs` and ships
|
||||
* nothing. Setting the key opts into the cloud dashboard (the consent boundary).
|
||||
* Runs under bun (the adapters use Bun.which), so invoke via `bun run`, never the
|
||||
* node-based `braintrust eval` CLI.
|
||||
*/
|
||||
import { Eval } from 'braintrust';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
import type { ProviderAdapter } from './providers/types';
|
||||
|
||||
export type ProviderName = 'claude' | 'gpt' | 'gemini';
|
||||
|
||||
const ADAPTERS: Record<ProviderName, () => ProviderAdapter> = {
|
||||
claude: () => new ClaudeAdapter(),
|
||||
gpt: () => new GptAdapter(),
|
||||
gemini: () => new GeminiAdapter(),
|
||||
};
|
||||
|
||||
export interface BenchCase {
|
||||
id: string;
|
||||
input: string;
|
||||
required: string[];
|
||||
}
|
||||
|
||||
export interface BenchOpts {
|
||||
timeoutMs?: number;
|
||||
/** Add an autoevals LLM judge (ClosedQA). Needs OPENAI_API_KEY. */
|
||||
judge?: boolean;
|
||||
}
|
||||
|
||||
/** Per-case operational metrics the adapter reports — Braintrust doesn't time the CLI for us. */
|
||||
export interface CaseOps {
|
||||
id: string;
|
||||
durationMs: number;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
costUsd: number;
|
||||
toolCalls: number;
|
||||
modelUsed: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ProviderBenchmark {
|
||||
provider: ProviderName;
|
||||
/** Mean required-terms score 0..1 from Braintrust, or null if every case errored. */
|
||||
score: number | null;
|
||||
ops: CaseOps[];
|
||||
}
|
||||
|
||||
export const DEFAULT_CORPUS_PATH = path.join(__dirname, '..', '..', 'evals', 'model-benchmark', 'corpus.json');
|
||||
|
||||
export function loadCorpus(corpusPath = DEFAULT_CORPUS_PATH): BenchCase[] {
|
||||
return JSON.parse(fs.readFileSync(corpusPath, 'utf-8'));
|
||||
}
|
||||
|
||||
interface ScorerArgs {
|
||||
input: string;
|
||||
output: string;
|
||||
expected: string[];
|
||||
}
|
||||
type ScoreResult = { name: string; score: number };
|
||||
type Scorer = (args: ScorerArgs) => ScoreResult | Promise<ScoreResult>;
|
||||
|
||||
/** Deterministic scorer: fraction of required terms present. Replaces the old in-house evaluation.ts. */
|
||||
const requiredTerms: Scorer = ({ output, expected }) => {
|
||||
const norm = (output ?? '').toLowerCase();
|
||||
const matched = (expected ?? []).filter((t) => norm.includes(t.toLowerCase()));
|
||||
return { name: 'required-terms', score: expected?.length ? matched.length / expected.length : 1 };
|
||||
};
|
||||
|
||||
/**
|
||||
* Run one provider across the cases through Braintrust and return its score + ops.
|
||||
* The adapter is the `task`; requiredTerms (and optionally an autoevals judge) are
|
||||
* the `scores`. Empty output with zero tokens is thrown so it can't fake a 0.
|
||||
*/
|
||||
export async function runProviderBenchmark(
|
||||
provider: ProviderName,
|
||||
cases: BenchCase[],
|
||||
opts: BenchOpts = {},
|
||||
): Promise<ProviderBenchmark> {
|
||||
const factory = ADAPTERS[provider];
|
||||
if (!factory) throw new Error(`unknown provider '${provider}' (claude|gpt|gemini)`);
|
||||
const adapter = factory();
|
||||
const timeoutMs = opts.timeoutMs ?? 300_000;
|
||||
|
||||
const ops: CaseOps[] = [];
|
||||
const byInput = new Map(cases.map((c) => [c.input, c]));
|
||||
|
||||
// Braintrust calls scorers with { input, output, expected, metadata }.
|
||||
const scores: Scorer[] = [requiredTerms];
|
||||
if (opts.judge) {
|
||||
// autoevals ClosedQA = Braintrust's own LLM-judge, replacing the in-house judge.ts.
|
||||
// Normalize its result to a plain { name, score } so cross-package Score types don't clash.
|
||||
const { ClosedQA } = await import('autoevals');
|
||||
const closedQa = ClosedQA as unknown as (a: Record<string, unknown>) => Promise<{ score?: number }>;
|
||||
scores.push(async ({ input, output, expected }) => {
|
||||
const r = await closedQa({
|
||||
input,
|
||||
output,
|
||||
criteria: `Addresses the task and mentions: ${(expected ?? []).join(', ')}`,
|
||||
});
|
||||
return { name: 'judge-closedqa', score: typeof r?.score === 'number' ? r.score : 0 };
|
||||
});
|
||||
}
|
||||
|
||||
const noSendLogs = !process.env.BRAINTRUST_API_KEY;
|
||||
|
||||
const result = await Eval(
|
||||
`gstack-model-benchmark:${provider}`,
|
||||
{
|
||||
data: () => cases.map((c) => ({ input: c.input, expected: c.required, metadata: { id: c.id } })),
|
||||
task: async (input: string) => {
|
||||
const c = byInput.get(input);
|
||||
const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-bench-'));
|
||||
try {
|
||||
const r = await adapter.run({ prompt: input, workdir, timeoutMs });
|
||||
const costUsd = adapter.estimateCost(r.tokens, r.modelUsed);
|
||||
const op: CaseOps = {
|
||||
id: c?.id ?? input.slice(0, 24),
|
||||
durationMs: r.durationMs,
|
||||
tokensIn: r.tokens.input,
|
||||
tokensOut: r.tokens.output,
|
||||
costUsd,
|
||||
toolCalls: r.toolCalls,
|
||||
modelUsed: r.modelUsed,
|
||||
error: r.error ? `${r.error.code}: ${r.error.reason}` : undefined,
|
||||
};
|
||||
ops.push(op);
|
||||
if (r.error) throw new Error(`${provider} failed: ${r.error.code} — ${r.error.reason}`);
|
||||
// Empty output + zero tokens = provider never answered (silent auth/CLI failure).
|
||||
// Fail loud so it can't masquerade as a 0.0 score.
|
||||
if (!r.output.trim() && r.tokens.input === 0 && r.tokens.output === 0) {
|
||||
throw new Error(`${provider} returned empty output with zero tokens (likely auth/CLI failure, not a real result)`);
|
||||
}
|
||||
return r.output;
|
||||
} finally {
|
||||
fs.rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
scores,
|
||||
},
|
||||
{ noSendLogs },
|
||||
);
|
||||
|
||||
const scoreSummary = result.summary?.scores?.['required-terms'];
|
||||
const score = typeof scoreSummary?.score === 'number' ? scoreSummary.score : null;
|
||||
return { provider, score, ops };
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
/**
|
||||
* Benchmark quality judge for multi-provider scoring.
|
||||
*
|
||||
* The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
|
||||
* the prompt + N provider outputs and scores each on: correctness, completeness,
|
||||
* code quality, edge case handling. 0-10 per dimension; overall = average.
|
||||
*
|
||||
* Judge adds ~$0.05 per benchmark run. Gated by --judge CLI flag.
|
||||
*/
|
||||
|
||||
import type { BenchmarkReport, BenchmarkEntry } from './runner';
|
||||
|
||||
export async function judgeEntries(report: BenchmarkReport): Promise<void> {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
throw new Error('ANTHROPIC_API_KEY not set — judge requires Anthropic access.');
|
||||
}
|
||||
const { default: Anthropic } = await import('@anthropic-ai/sdk').catch(() => {
|
||||
throw new Error('@anthropic-ai/sdk not installed — run `bun add @anthropic-ai/sdk` if you want the judge.');
|
||||
});
|
||||
const client = new (Anthropic as unknown as new (opts: { apiKey: string }) => {
|
||||
messages: { create: (params: Record<string, unknown>) => Promise<{ content: Array<{ type: string; text: string }> }> };
|
||||
})({ apiKey: process.env.ANTHROPIC_API_KEY! });
|
||||
|
||||
const successful = report.entries.filter(e => e.available && e.result && !e.result.error);
|
||||
if (successful.length === 0) return;
|
||||
|
||||
const judgePrompt = buildJudgePrompt(report.prompt, successful);
|
||||
const msg = await client.messages.create({
|
||||
model: 'claude-sonnet-4-6',
|
||||
max_tokens: 2048,
|
||||
messages: [{ role: 'user', content: judgePrompt }],
|
||||
});
|
||||
const textBlock = msg.content.find(c => c.type === 'text');
|
||||
if (!textBlock) return;
|
||||
|
||||
const scores = parseScores(textBlock.text, successful.length);
|
||||
for (let i = 0; i < successful.length; i++) {
|
||||
const s = scores[i];
|
||||
if (!s) continue;
|
||||
successful[i].qualityScore = s.overall;
|
||||
successful[i].qualityDetails = s.dimensions;
|
||||
}
|
||||
}
|
||||
|
||||
function buildJudgePrompt(prompt: string, entries: BenchmarkEntry[]): string {
|
||||
const lines: string[] = [
|
||||
'You are a strict, fair technical reviewer scoring N model outputs against the same prompt.',
|
||||
'',
|
||||
'--- PROMPT ---',
|
||||
prompt.length > 4000 ? prompt.slice(0, 4000) + '\n[...truncated for judge budget...]' : prompt,
|
||||
'',
|
||||
'--- OUTPUTS ---',
|
||||
];
|
||||
entries.forEach((e, i) => {
|
||||
const r = e.result!;
|
||||
const out = r.output.length > 3000 ? r.output.slice(0, 3000) + '\n[...truncated...]' : r.output;
|
||||
lines.push(`=== Output ${i + 1}: ${r.modelUsed} ===`);
|
||||
lines.push(out);
|
||||
lines.push('');
|
||||
});
|
||||
lines.push('');
|
||||
lines.push('Score each output on these dimensions (0-10 per dimension):');
|
||||
lines.push(' - correctness: does it solve what the prompt asked?');
|
||||
lines.push(' - completeness: are edge cases and error paths addressed?');
|
||||
lines.push(' - code_quality: naming, structure, explicitness');
|
||||
lines.push(' - edge_cases: handling of nil/empty/invalid input');
|
||||
lines.push('');
|
||||
lines.push('Return JSON only, in this exact shape:');
|
||||
lines.push('{"scores":[');
|
||||
lines.push(' {"output":1,"correctness":N,"completeness":N,"code_quality":N,"edge_cases":N,"overall":N,"notes":"..."},');
|
||||
lines.push(' ...');
|
||||
lines.push(']}');
|
||||
lines.push('');
|
||||
lines.push('overall = rounded average of the 4 dimensions. No other commentary.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
interface ParsedScore {
|
||||
overall: number;
|
||||
dimensions: Record<string, number>;
|
||||
}
|
||||
|
||||
function parseScores(raw: string, expectedCount: number): ParsedScore[] {
|
||||
const match = raw.match(/\{[\s\S]*\}/);
|
||||
if (!match) return [];
|
||||
try {
|
||||
const obj = JSON.parse(match[0]);
|
||||
if (!Array.isArray(obj.scores)) return [];
|
||||
return obj.scores.slice(0, expectedCount).map((s: Record<string, number>) => ({
|
||||
overall: Number(s.overall ?? 0),
|
||||
dimensions: {
|
||||
correctness: Number(s.correctness ?? 0),
|
||||
completeness: Number(s.completeness ?? 0),
|
||||
code_quality: Number(s.code_quality ?? 0),
|
||||
edge_cases: Number(s.edge_cases ?? 0),
|
||||
},
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
/**
|
||||
* Multi-provider benchmark runner.
|
||||
*
|
||||
* Orchestrates running the same prompt across multiple provider adapters and
|
||||
* aggregates RunResult outputs + judge scores into a single report. Adapters
|
||||
* run in parallel (Promise.allSettled) so a slow provider doesn't block a fast
|
||||
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
|
||||
*/
|
||||
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
|
||||
export interface BenchmarkInput {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
timeoutMs?: number;
|
||||
/** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */
|
||||
providers: Array<'claude' | 'gpt' | 'gemini'>;
|
||||
/** Optional per-provider model overrides. */
|
||||
models?: Partial<Record<'claude' | 'gpt' | 'gemini', string>>;
|
||||
/** If true, skip providers whose available() returns !ok. If false, include them with error. */
|
||||
skipUnavailable?: boolean;
|
||||
}
|
||||
|
||||
export interface BenchmarkEntry {
|
||||
provider: string;
|
||||
family: 'claude' | 'gpt' | 'gemini';
|
||||
available: boolean;
|
||||
unavailable_reason?: string;
|
||||
result?: RunResult;
|
||||
costUsd?: number;
|
||||
/** Judge score 0-10 across dimensions. Populated separately by the judge step. */
|
||||
qualityScore?: number;
|
||||
qualityDetails?: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface BenchmarkReport {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
startedAt: string;
|
||||
durationMs: number;
|
||||
entries: BenchmarkEntry[];
|
||||
}
|
||||
|
||||
const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = {
|
||||
claude: () => new ClaudeAdapter(),
|
||||
gpt: () => new GptAdapter(),
|
||||
gemini: () => new GeminiAdapter(),
|
||||
};
|
||||
|
||||
export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkReport> {
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const timeoutMs = input.timeoutMs ?? 300_000;
|
||||
|
||||
const entries: BenchmarkEntry[] = [];
|
||||
const runPromises: Array<Promise<void>> = [];
|
||||
|
||||
for (const name of input.providers) {
|
||||
const factory = ADAPTERS[name];
|
||||
if (!factory) {
|
||||
entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` });
|
||||
continue;
|
||||
}
|
||||
const adapter = factory();
|
||||
const entry: BenchmarkEntry = { provider: adapter.name, family: adapter.family, available: true };
|
||||
entries.push(entry);
|
||||
|
||||
runPromises.push((async () => {
|
||||
const check = await adapter.available();
|
||||
entry.available = check.ok;
|
||||
if (!check.ok) {
|
||||
entry.unavailable_reason = check.reason;
|
||||
if (input.skipUnavailable) return;
|
||||
}
|
||||
const opts: RunOpts = {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
timeoutMs,
|
||||
model: input.models?.[name],
|
||||
};
|
||||
const res = await adapter.run(opts);
|
||||
entry.result = res;
|
||||
entry.costUsd = adapter.estimateCost(res.tokens, res.modelUsed);
|
||||
})());
|
||||
}
|
||||
|
||||
await Promise.allSettled(runPromises);
|
||||
|
||||
return {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
startedAt,
|
||||
durationMs: Date.now() - startedAtMs,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
|
||||
export function formatJson(report: BenchmarkReport): string {
|
||||
return JSON.stringify(report, null, 2);
|
||||
}
|
||||
|
||||
export function formatMarkdown(report: BenchmarkReport): string {
|
||||
const lines: string[] = [
|
||||
`# Benchmark report — ${report.startedAt}`,
|
||||
'',
|
||||
`**Prompt:** ${report.prompt.length > 200 ? report.prompt.slice(0, 200) + '…' : report.prompt}`,
|
||||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
lines.push(`| ${e.provider} | - | - | - | - | - | unavailable: ${e.unavailable_reason ?? 'unknown'} |`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function msToStr(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function fmtCost(usd?: number): string {
|
||||
if (usd === undefined) return '-';
|
||||
if (usd < 0.01) return `$${usd.toFixed(4)}`;
|
||||
return `$${usd.toFixed(2)}`;
|
||||
}
|
||||
Reference in New Issue
Block a user