CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-21 04:10:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: capture plain-text CLI output, drop hardcoded provider schemas

Browse Source 
The adapters parsed each vendor's proprietary JSON stream (Claude json,
Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a
per-model pricing table turned tokens into cost. That coupling was the
brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor
reshuffled its output, and it duplicated what any tool that instruments the
real model call already does. Braintrust owns scoring; it can't see a CLI
subprocess's tokens anyway, so computing cost ourselves meant maintaining
both a parser and a price table forever.

Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring
is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls;
the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all
three JSON parsers, and the Gemini stream-schema parser + its test. Gemini
auth detection (env/OAuth/.env) is kept — that's not schema parsing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sinabina
2026-07-21 14:34:57 -07:00
co-authored by Claude Opus 4.8
parent 36f972f2e4
commit 7f4574e1d5
12 changed files with 84 additions and 441 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/gstack-model-benchmark
+2 -10
View File
@@ -128,18 +128,15 @@ async function main(): Promise<void> {
}
function formatTable(results: ProviderBenchmark[]): string {
const header = `Provider Score Cases Latency(avg) Tokens(in→out) Cost Errors`;
const header = `Provider Score Cases Latency(avg) Errors`;
const rows: string[] = [header, '-'.repeat(header.length)];
for (const r of results) {
const n = r.ops.length || 1;
const avgMs = r.ops.reduce((s, o) => s + o.durationMs, 0) / n;
const tin = r.ops.reduce((s, o) => s + o.tokensIn, 0);
const tout = r.ops.reduce((s, o) => s + o.tokensOut, 0);
const cost = r.ops.reduce((s, o) => s + o.costUsd, 0);
const errs = r.ops.filter(o => o.error).length;
const score = r.score === null ? '—' : `${(r.score * 100).toFixed(1)}%`;
rows.push(
`${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${pad(`${tin}→${tout}`, 15)} ${pad(fmtCost(cost), 9)} ${errs || ''}`,
`${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${errs || ''}`,
);
}
return rows.join('\n');
@@ -192,11 +189,6 @@ function msToStr(ms: number): string {
if (ms < 1000) return `${Math.round(ms)}ms`;
return `${(ms / 1000).toFixed(1)}s`;
}
function fmtCost(usd: number): string {
if (usd === 0) return '$0';
if (usd < 0.01) return `$${usd.toFixed(4)}`;
return `$${usd.toFixed(2)}`;
}
main().catch(err => {
console.error('FATAL:', err);