mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 20:30:47 +02:00
refactor: capture plain-text CLI output, drop hardcoded provider schemas
The adapters parsed each vendor's proprietary JSON stream (Claude json, Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a per-model pricing table turned tokens into cost. That coupling was the brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor reshuffled its output, and it duplicated what any tool that instruments the real model call already does. Braintrust owns scoring; it can't see a CLI subprocess's tokens anyway, so computing cost ourselves meant maintaining both a parser and a price table forever. Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls; the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all three JSON parsers, and the Gemini stream-schema parser + its test. Gemini auth detection (env/OAuth/.env) is kept — that's not schema parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sinabina
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
36f972f2e4
commit
7f4574e1d5
@@ -1,18 +1,15 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
* Gemini adapter — wraps the `gemini` CLI in plain-text mode.
|
||||
*
|
||||
* Gemini CLI auth comes from its OAuth files, ~/.gemini/.env, or the
|
||||
* GOOGLE_API_KEY/GEMINI_API_KEY environment variables. Output
|
||||
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
|
||||
* stream-json` is requested. This adapter uses a single-response form for simplicity
|
||||
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
|
||||
* Auth comes from its OAuth files, ~/.gemini/.env, or GOOGLE_API_KEY/GEMINI_API_KEY.
|
||||
* We capture stdout as the answer and do not parse Gemini's stream-json schema —
|
||||
* that coupling broke every time Gemini reshuffled its event shape.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
readonly name = 'gemini';
|
||||
@@ -39,9 +36,10 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
// Benchmarks are report-only. Plan mode keeps the CLI non-interactive while
|
||||
// preventing a benchmark prompt from mutating the target worktree.
|
||||
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--approval-mode', 'plan', '--skip-trust'];
|
||||
// --skip-trust lets the CLI run in disposable/non-git benchmark workdirs.
|
||||
// Plain -p is non-interactive; benchmark prompts are reasoning tasks, not file
|
||||
// ops, and the workdir is a throwaway temp dir.
|
||||
const args = ['-p', opts.prompt, '--skip-trust'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
@@ -52,83 +50,25 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = parseGeminiStreamJson(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
output: out.trim(),
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
|
||||
modelUsed: opts.model ?? 'gemini',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login|api key/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.errorResult(Date.now() - start, err, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gemini-2.5-pro',
|
||||
error,
|
||||
};
|
||||
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
let code: RunError;
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
||||
else if (/unauthorized|auth|login|api key/i.test(stderr)) code = 'auth';
|
||||
else if (/rate[- ]?limit|429|quota/i.test(stderr)) code = 'rate_limit';
|
||||
else code = 'unknown';
|
||||
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
||||
return { output: '', durationMs, modelUsed: model ?? 'gemini', error: { code, reason } };
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse both legacy `usage` events and current Gemini CLI `stats` events. */
|
||||
export function parseGeminiStreamJson(raw: string): {
|
||||
output: string;
|
||||
tokens: { input: number; output: number };
|
||||
toolCalls: number;
|
||||
modelUsed?: string;
|
||||
} {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'init' && typeof obj.model === 'string' && obj.model !== 'auto') {
|
||||
modelUsed = obj.model;
|
||||
} else if (obj.type === 'message' && obj.role === 'assistant') {
|
||||
const content = typeof obj.content === 'string' ? obj.content : obj.text;
|
||||
if (typeof content === 'string') output += content;
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? obj.stats ?? {};
|
||||
input += u.input_token_count ?? u.prompt_tokens ?? u.input_tokens ?? u.input ?? 0;
|
||||
out += u.output_token_count ?? u.completion_tokens ?? u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
if (!modelUsed && u.models && typeof u.models === 'object') {
|
||||
modelUsed = Object.keys(u.models)[0];
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user