mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 20:30:47 +02:00
SinabinaandClaude Opus 4.8
7f4574e1d5
The adapters parsed each vendor's proprietary JSON stream (Claude json, Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a per-model pricing table turned tokens into cost. That coupling was the brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor reshuffled its output, and it duplicated what any tool that instruments the real model call already does. Braintrust owns scoring; it can't see a CLI subprocess's tokens anyway, so computing cost ourselves meant maintaining both a parser and a price table forever. Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls; the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all three JSON parsers, and the Gemini stream-schema parser + its test. Gemini auth detection (env/OAuth/.env) is kept — that's not schema parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
64 lines
2.1 KiB
TypeScript
64 lines
2.1 KiB
TypeScript
import { expect, test } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const PRODUCTION_ROOTS = [path.join(ROOT, 'bin'), path.join(ROOT, 'lib')];
|
|
const IMPORT_SPECIFIER = /(?:\bfrom\s*|\bimport\s*(?:\(\s*)?|\brequire\s*\(\s*)['"]([^'"]+)['"]/g;
|
|
const TEST_SEGMENT = /(?:^|\/)tests?(?:\/|$)/;
|
|
|
|
function sourceFiles(dir: string): string[] {
|
|
const files: string[] = [];
|
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
const absolute = path.join(dir, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (entry.name === 'dist' || entry.name === 'node_modules') continue;
|
|
files.push(...sourceFiles(absolute));
|
|
continue;
|
|
}
|
|
if (!entry.isFile()) continue;
|
|
if (dir === path.join(ROOT, 'bin') || /\.(?:[cm]?[jt]s|tsx)$/.test(entry.name)) {
|
|
files.push(absolute);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
test('production modules do not import from test directories', () => {
|
|
const violations: string[] = [];
|
|
|
|
for (const file of PRODUCTION_ROOTS.flatMap(sourceFiles)) {
|
|
const source = fs.readFileSync(file, 'utf8');
|
|
for (const match of source.matchAll(IMPORT_SPECIFIER)) {
|
|
const specifier = match[1].replaceAll('\\', '/');
|
|
if (TEST_SEGMENT.test(specifier)) {
|
|
violations.push(`${path.relative(ROOT, file)} -> ${match[1]}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
expect(violations).toEqual([]);
|
|
});
|
|
|
|
test('former test-helper paths re-export the production benchmark API', async () => {
|
|
const [
|
|
claude,
|
|
helperClaude,
|
|
gpt,
|
|
helperGpt,
|
|
gemini,
|
|
helperGemini,
|
|
] = await Promise.all([
|
|
import('../lib/model-benchmark/providers/claude'),
|
|
import('./helpers/providers/claude'),
|
|
import('../lib/model-benchmark/providers/gpt'),
|
|
import('./helpers/providers/gpt'),
|
|
import('../lib/model-benchmark/providers/gemini'),
|
|
import('./helpers/providers/gemini'),
|
|
]);
|
|
|
|
expect(helperClaude.ClaudeAdapter).toBe(claude.ClaudeAdapter);
|
|
expect(helperGpt.GptAdapter).toBe(gpt.GptAdapter);
|
|
expect(helperGemini.GeminiAdapter).toBe(gemini.GeminiAdapter);
|
|
});
|