mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 20:30:47 +02:00
SinabinaandClaude Opus 4.8
36f972f2e4
Braintrust now owns benchmark scoring, experiments, comparison, and reporting. GStack keeps only the CLI-agent adapters (the unavoidable shim) plus operational metrics the CLIs report. runProviderBenchmark wraps each adapter as a Braintrust Eval task; a deterministic required-terms scorer replaces the in-house evaluation logic and the optional autoevals ClosedQA judge replaces judge.ts. Runs local by default under bun: with no BRAINTRUST_API_KEY it sets noSendLogs and ships nothing; setting the key opts into the cloud dashboard. An empty output with zero tokens is thrown so a silent auth/CLI failure can't masquerade as a 0.0 score. Deletes runner.ts and judge.ts (and their test-helper re-export shims); rewires bin/gstack-model-benchmark and adapts the benchmark tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
69 lines
2.2 KiB
TypeScript
69 lines
2.2 KiB
TypeScript
import { expect, test } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const PRODUCTION_ROOTS = [path.join(ROOT, 'bin'), path.join(ROOT, 'lib')];
|
|
const IMPORT_SPECIFIER = /(?:\bfrom\s*|\bimport\s*(?:\(\s*)?|\brequire\s*\(\s*)['"]([^'"]+)['"]/g;
|
|
const TEST_SEGMENT = /(?:^|\/)tests?(?:\/|$)/;
|
|
|
|
function sourceFiles(dir: string): string[] {
|
|
const files: string[] = [];
|
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
const absolute = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (entry.name === 'dist' || entry.name === 'node_modules') continue;
|
|
files.push(...sourceFiles(absolute));
|
|
continue;
|
|
}
|
|
if (!entry.isFile()) continue;
|
|
if (dir === path.join(ROOT, 'bin') || /\.(?:[cm]?[jt]s|tsx)$/.test(entry.name)) {
|
|
files.push(absolute);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
test('production modules do not import from test directories', () => {
|
|
const violations: string[] = [];
|
|
|
|
for (const file of PRODUCTION_ROOTS.flatMap(sourceFiles)) {
|
|
const source = fs.readFileSync(file, 'utf8');
|
|
for (const match of source.matchAll(IMPORT_SPECIFIER)) {
|
|
const specifier = match[1].replaceAll('\\', '/');
|
|
if (TEST_SEGMENT.test(specifier)) {
|
|
violations.push(`${path.relative(ROOT, file)} -> ${match[1]}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
expect(violations).toEqual([]);
|
|
});
|
|
|
|
test('former test-helper paths re-export the production benchmark API', async () => {
|
|
const [
|
|
pricing,
|
|
helperPricing,
|
|
claude,
|
|
helperClaude,
|
|
gpt,
|
|
helperGpt,
|
|
gemini,
|
|
helperGemini,
|
|
] = await Promise.all([
|
|
import('../lib/model-benchmark/pricing'),
|
|
import('./helpers/pricing'),
|
|
import('../lib/model-benchmark/providers/claude'),
|
|
import('./helpers/providers/claude'),
|
|
import('../lib/model-benchmark/providers/gpt'),
|
|
import('./helpers/providers/gpt'),
|
|
import('../lib/model-benchmark/providers/gemini'),
|
|
import('./helpers/providers/gemini'),
|
|
]);
|
|
|
|
expect(helperPricing.estimateCostUsd).toBe(pricing.estimateCostUsd);
|
|
expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
|
|
expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
|
|
expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);
|
|
});
|