mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 12:20:48 +02:00
SinabinaandClaude Opus 4.8
7f4574e1d5
The adapters parsed each vendor's proprietary JSON stream (Claude json, Codex JSONL, Gemini stream-json) to extract tokens/tool-calls, and a per-model pricing table turned tokens into cost. That coupling was the brittle hardcoding GStack 2 exists to avoid — it broke every time a vendor reshuffled its output, and it duplicated what any tool that instruments the real model call already does. Braintrust owns scoring; it can't see a CLI subprocess's tokens anyway, so computing cost ourselves meant maintaining both a parser and a price table forever. Now each adapter runs the CLI in plain-text mode and returns stdout. Scoring is unchanged (Braintrust reads the text). RunResult drops tokens/toolCalls; the comparison table drops the Tokens/Cost columns. Deletes pricing.ts, all three JSON parsers, and the Gemini stream-schema parser + its test. Gemini auth detection (env/OAuth/.env) is kept — that's not schema parsing. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
78 lines
3.3 KiB
TypeScript
78 lines
3.3 KiB
TypeScript
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck, RunError } from './types';
|
|
import { execFileSync } from 'child_process';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
|
|
|
|
/**
|
|
* Claude adapter — wraps the `claude` CLI via `claude -p` in plain-text mode.
|
|
*
|
|
* We capture stdout as the answer and do not parse Claude's JSON output shape:
|
|
* scoring is Braintrust's job and it only needs the text.
|
|
*/
|
|
export class ClaudeAdapter implements ProviderAdapter {
|
|
readonly name = 'claude';
|
|
readonly family = 'claude' as const;
|
|
|
|
async available(): Promise<AvailabilityCheck> {
|
|
// Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
|
|
// resolver so Windows + override paths behave the same as production sites.
|
|
const resolved = resolveClaudeCommand();
|
|
if (!resolved) {
|
|
return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
|
|
}
|
|
// Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY
|
|
const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
|
|
const hasCreds = fs.existsSync(credsPath);
|
|
const hasKey = !!process.env.ANTHROPIC_API_KEY;
|
|
if (!hasCreds && !hasKey) {
|
|
return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
|
|
}
|
|
return { ok: true };
|
|
}
|
|
|
|
async run(opts: RunOpts): Promise<RunResult> {
|
|
const start = Date.now();
|
|
const resolved = resolveClaudeCommand();
|
|
if (!resolved) {
|
|
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
|
|
}
|
|
const args = [...resolved.argsPrefix, '-p'];
|
|
if (opts.model) args.push('--model', opts.model);
|
|
if (opts.extraArgs) args.push(...opts.extraArgs);
|
|
|
|
try {
|
|
const out = execFileSync(resolved.command, args, {
|
|
input: opts.prompt,
|
|
cwd: opts.workdir,
|
|
timeout: opts.timeoutMs,
|
|
encoding: 'utf-8',
|
|
maxBuffer: 32 * 1024 * 1024,
|
|
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
|
|
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
|
|
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
|
});
|
|
return {
|
|
output: out.trim(),
|
|
durationMs: Date.now() - start,
|
|
modelUsed: opts.model ?? 'claude',
|
|
};
|
|
} catch (err: unknown) {
|
|
return this.errorResult(Date.now() - start, err, opts.model);
|
|
}
|
|
}
|
|
|
|
private errorResult(durationMs: number, err: unknown, model?: string): RunResult {
|
|
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
|
const stderr = e.stderr?.toString() ?? '';
|
|
let code: RunError;
|
|
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') code = 'timeout';
|
|
else if (/unauthorized|auth|login/i.test(stderr)) code = 'auth';
|
|
else if (/rate[- ]?limit|429/i.test(stderr)) code = 'rate_limit';
|
|
else code = 'unknown';
|
|
const reason = code === 'timeout' ? 'exceeded timeout' : (e.message ?? stderr ?? 'unknown').slice(0, 400);
|
|
return { output: '', durationMs, modelUsed: model ?? 'claude', error: { code, reason } };
|
|
}
|
|
}
|