mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 04:10:47 +02:00
implement six-skill gstack 2 runtime
This commit is contained in:
Sinabina
+3
-10
@@ -1,19 +1,12 @@
|
||||
/**
|
||||
* bin-context — tiny shared helpers for non-interactive gstack bins that need the
|
||||
* project slug, current branch, and argv flags. Extracted from the decision bins
|
||||
* (gstack-decision-log / gstack-decision-search) so the slug/branch/flag plumbing
|
||||
* lives in one audited place instead of being copy-pasted per bin.
|
||||
* current branch and argv flags. Project identity is resolved directly through
|
||||
* runtime/identity.js by callers so native Windows never has to execute a
|
||||
* sibling shebang script.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "child_process";
|
||||
|
||||
/** Resolve the project slug via the `gstack-slug` helper (parses `SLUG=...`). */
|
||||
export function resolveSlug(slugBinPath: string): string {
|
||||
const r = spawnSync(slugBinPath, { encoding: "utf-8" });
|
||||
const m = (r.stdout || "").match(/^SLUG=(.+)$/m);
|
||||
return m ? m[1].trim() : "unknown";
|
||||
}
|
||||
|
||||
/** Current git branch, or undefined on detached HEAD / outside a repo. */
|
||||
export function gitBranch(): string | undefined {
|
||||
const r = spawnSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], { encoding: "utf-8" });
|
||||
|
||||
@@ -54,10 +54,10 @@ export interface DecisionPaths {
|
||||
archive: string;
|
||||
}
|
||||
|
||||
/** Resolve the per-project decision store paths. Bins pass slug + GSTACK_HOME. */
|
||||
export function decisionPaths(slug: string, gstackHome?: string): DecisionPaths {
|
||||
/** Resolve the per-worktree decision store paths. Bins pass projectId + GSTACK_HOME. */
|
||||
export function decisionPaths(projectId: string, gstackHome?: string): DecisionPaths {
|
||||
const home = gstackHome || process.env.GSTACK_HOME || join(homedir(), ".gstack");
|
||||
const dir = join(home, "projects", slug || "unknown");
|
||||
const dir = join(home, "projects", projectId || "unknown");
|
||||
return {
|
||||
log: join(dir, "decisions.jsonl"),
|
||||
snapshot: join(dir, "decisions.active.json"),
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Benchmark quality judge for multi-provider scoring.
|
||||
*
|
||||
* The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
|
||||
* the prompt + N provider outputs and scores each on: correctness, completeness,
|
||||
* code quality, edge case handling. 0-10 per dimension; overall = average.
|
||||
*
|
||||
* Judge adds ~$0.05 per benchmark run. Gated by --judge CLI flag.
|
||||
*/
|
||||
|
||||
import type { BenchmarkReport, BenchmarkEntry } from './runner';
|
||||
|
||||
export async function judgeEntries(report: BenchmarkReport): Promise<void> {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
throw new Error('ANTHROPIC_API_KEY not set — judge requires Anthropic access.');
|
||||
}
|
||||
const { default: Anthropic } = await import('@anthropic-ai/sdk').catch(() => {
|
||||
throw new Error('@anthropic-ai/sdk not installed — run `bun add @anthropic-ai/sdk` if you want the judge.');
|
||||
});
|
||||
const client = new (Anthropic as unknown as new (opts: { apiKey: string }) => {
|
||||
messages: { create: (params: Record<string, unknown>) => Promise<{ content: Array<{ type: string; text: string }> }> };
|
||||
})({ apiKey: process.env.ANTHROPIC_API_KEY! });
|
||||
|
||||
const successful = report.entries.filter(e => e.available && e.result && !e.result.error);
|
||||
if (successful.length === 0) return;
|
||||
|
||||
const judgePrompt = buildJudgePrompt(report.prompt, successful);
|
||||
const msg = await client.messages.create({
|
||||
model: 'claude-sonnet-4-6',
|
||||
max_tokens: 2048,
|
||||
messages: [{ role: 'user', content: judgePrompt }],
|
||||
});
|
||||
const textBlock = msg.content.find(c => c.type === 'text');
|
||||
if (!textBlock) return;
|
||||
|
||||
const scores = parseScores(textBlock.text, successful.length);
|
||||
for (let i = 0; i < successful.length; i++) {
|
||||
const s = scores[i];
|
||||
if (!s) continue;
|
||||
successful[i].qualityScore = s.overall;
|
||||
successful[i].qualityDetails = s.dimensions;
|
||||
}
|
||||
}
|
||||
|
||||
function buildJudgePrompt(prompt: string, entries: BenchmarkEntry[]): string {
|
||||
const lines: string[] = [
|
||||
'You are a strict, fair technical reviewer scoring N model outputs against the same prompt.',
|
||||
'',
|
||||
'--- PROMPT ---',
|
||||
prompt.length > 4000 ? prompt.slice(0, 4000) + '\n[...truncated for judge budget...]' : prompt,
|
||||
'',
|
||||
'--- OUTPUTS ---',
|
||||
];
|
||||
entries.forEach((e, i) => {
|
||||
const r = e.result!;
|
||||
const out = r.output.length > 3000 ? r.output.slice(0, 3000) + '\n[...truncated...]' : r.output;
|
||||
lines.push(`=== Output ${i + 1}: ${r.modelUsed} ===`);
|
||||
lines.push(out);
|
||||
lines.push('');
|
||||
});
|
||||
lines.push('');
|
||||
lines.push('Score each output on these dimensions (0-10 per dimension):');
|
||||
lines.push(' - correctness: does it solve what the prompt asked?');
|
||||
lines.push(' - completeness: are edge cases and error paths addressed?');
|
||||
lines.push(' - code_quality: naming, structure, explicitness');
|
||||
lines.push(' - edge_cases: handling of nil/empty/invalid input');
|
||||
lines.push('');
|
||||
lines.push('Return JSON only, in this exact shape:');
|
||||
lines.push('{"scores":[');
|
||||
lines.push(' {"output":1,"correctness":N,"completeness":N,"code_quality":N,"edge_cases":N,"overall":N,"notes":"..."},');
|
||||
lines.push(' ...');
|
||||
lines.push(']}');
|
||||
lines.push('');
|
||||
lines.push('overall = rounded average of the 4 dimensions. No other commentary.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
interface ParsedScore {
|
||||
overall: number;
|
||||
dimensions: Record<string, number>;
|
||||
}
|
||||
|
||||
function parseScores(raw: string, expectedCount: number): ParsedScore[] {
|
||||
const match = raw.match(/\{[\s\S]*\}/);
|
||||
if (!match) return [];
|
||||
try {
|
||||
const obj = JSON.parse(match[0]);
|
||||
if (!Array.isArray(obj.scores)) return [];
|
||||
return obj.scores.slice(0, expectedCount).map((s: Record<string, number>) => ({
|
||||
overall: Number(s.overall ?? 0),
|
||||
dimensions: {
|
||||
correctness: Number(s.correctness ?? 0),
|
||||
completeness: Number(s.completeness ?? 0),
|
||||
code_quality: Number(s.code_quality ?? 0),
|
||||
edge_cases: Number(s.edge_cases ?? 0),
|
||||
},
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Per-model pricing tables.
|
||||
*
|
||||
* Prices are USD per million tokens as of `as_of`. Update quarterly.
|
||||
* Link to provider pricing pages:
|
||||
* - Anthropic: https://www.anthropic.com/pricing#api
|
||||
* - OpenAI: https://openai.com/api/pricing/
|
||||
* - Google AI: https://ai.google.dev/pricing
|
||||
*
|
||||
* When a model isn't in the table, estimateCost returns 0 with a console warning.
|
||||
* Prefer adding a new row to the table over guessing.
|
||||
*/
|
||||
|
||||
export interface ModelPricing {
|
||||
input_per_mtok: number;
|
||||
output_per_mtok: number;
|
||||
as_of: string; // YYYY-MM
|
||||
}
|
||||
|
||||
export const PRICING: Record<string, ModelPricing> = {
|
||||
// Claude (Anthropic)
|
||||
'claude-opus-4-7': { input_per_mtok: 15.00, output_per_mtok: 75.00, as_of: '2026-04' },
|
||||
'claude-sonnet-4-6': { input_per_mtok: 3.00, output_per_mtok: 15.00, as_of: '2026-04' },
|
||||
'claude-haiku-4-5': { input_per_mtok: 1.00, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
|
||||
// OpenAI (GPT + o-series)
|
||||
'gpt-5.4': { input_per_mtok: 2.50, output_per_mtok: 10.00, as_of: '2026-04' },
|
||||
'gpt-5.4-mini': { input_per_mtok: 0.60, output_per_mtok: 2.40, as_of: '2026-04' },
|
||||
'o3': { input_per_mtok: 15.00, output_per_mtok: 60.00, as_of: '2026-04' },
|
||||
'o4-mini': { input_per_mtok: 1.10, output_per_mtok: 4.40, as_of: '2026-04' },
|
||||
|
||||
// Google
|
||||
'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' },
|
||||
'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' },
|
||||
};
|
||||
|
||||
const WARNED = new Set<string>();
|
||||
|
||||
export function estimateCostUsd(
|
||||
tokens: { input: number; output: number; cached?: number },
|
||||
model: string | undefined
|
||||
): number {
|
||||
if (!model) return 0;
|
||||
const row = PRICING[model];
|
||||
if (!row) {
|
||||
if (!WARNED.has(model)) {
|
||||
WARNED.add(model);
|
||||
console.error(`WARN: no pricing for model ${model}; returning 0. Add it to lib/model-benchmark/pricing.ts.`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
// Anthropic and OpenAI report cached tokens as a separate (disjoint) field from
|
||||
// uncached input tokens. tokens.input is already the uncached portion; tokens.cached
|
||||
// is the cache-read count billed at 10% of the regular input rate. Do NOT subtract
|
||||
// cached from input — they don't overlap.
|
||||
const cachedDiscount = 0.1;
|
||||
const inputCost = tokens.input * row.input_per_mtok / 1_000_000;
|
||||
const cachedCost = (tokens.cached ?? 0) * row.input_per_mtok * cachedDiscount / 1_000_000;
|
||||
const outputCost = tokens.output * row.output_per_mtok / 1_000_000;
|
||||
return +(inputCost + cachedCost + outputCost).toFixed(6);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import { execFileSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
|
||||
|
||||
/**
|
||||
* Claude adapter — wraps the `claude` CLI via claude -p.
|
||||
*
|
||||
* For brevity and to avoid duplicating the full stream-json parser, this adapter
|
||||
* uses claude CLI in non-interactive mode (--print) with the simpler JSON output
|
||||
* format. If richer event-level metrics are needed (per-tool timing etc.),
|
||||
* swap to session-runner's full stream-json parser.
|
||||
*/
|
||||
export class ClaudeAdapter implements ProviderAdapter {
|
||||
readonly name = 'claude';
|
||||
readonly family = 'claude' as const;
|
||||
|
||||
async available(): Promise<AvailabilityCheck> {
|
||||
// Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
|
||||
// resolver so Windows + override paths behave the same as production sites.
|
||||
const resolved = resolveClaudeCommand();
|
||||
if (!resolved) {
|
||||
return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
|
||||
}
|
||||
// Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY
|
||||
const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
|
||||
const hasCreds = fs.existsSync(credsPath);
|
||||
const hasKey = !!process.env.ANTHROPIC_API_KEY;
|
||||
if (!hasCreds && !hasKey) {
|
||||
return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
const resolved = resolveClaudeCommand();
|
||||
if (!resolved) {
|
||||
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
|
||||
}
|
||||
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
try {
|
||||
const out = execFileSync(resolved.command, args, {
|
||||
input: opts.prompt,
|
||||
cwd: opts.workdir,
|
||||
timeout: opts.timeoutMs,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
|
||||
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
||||
});
|
||||
const parsed = this.parseOutput(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse claude -p --output-format json output. Shape (as of 2026-04):
|
||||
* { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
|
||||
* num_turns, session_id, ... }
|
||||
* Older formats may differ — adapter is best-effort.
|
||||
*/
|
||||
private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
|
||||
try {
|
||||
const obj = JSON.parse(raw);
|
||||
const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
|
||||
const u = obj.usage ?? {};
|
||||
return {
|
||||
output: result,
|
||||
tokens: {
|
||||
input: u.input_tokens ?? 0,
|
||||
output: u.output_tokens ?? 0,
|
||||
cached: u.cache_read_input_tokens,
|
||||
},
|
||||
toolCalls: obj.num_turns ?? 0,
|
||||
modelUsed: obj.model,
|
||||
};
|
||||
} catch {
|
||||
// Non-JSON output: treat as plain text.
|
||||
return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'claude-opus-4-7',
|
||||
error,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
*
|
||||
* Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
|
||||
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
|
||||
* stream-json` is requested. This adapter uses a single-response form for simplicity
|
||||
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
readonly name = 'gemini';
|
||||
readonly family = 'gemini' as const;
|
||||
|
||||
async available(): Promise<AvailabilityCheck> {
|
||||
const res = spawnSync('sh', ['-c', 'command -v gemini'], { timeout: 2000 });
|
||||
if (res.status !== 0) {
|
||||
return { ok: false, reason: 'gemini CLI not found on PATH. Install per https://github.com/google-gemini/gemini-cli' };
|
||||
}
|
||||
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
|
||||
const newCfgDir = path.join(os.homedir(), '.gemini');
|
||||
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
|
||||
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
|
||||
const hasKey = !!process.env.GOOGLE_API_KEY;
|
||||
if (!hasCfg && !hasKey) {
|
||||
return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
// Default to --yolo (non-interactive) and stream-json output so we can parse
|
||||
// tokens + tool calls. Callers can override via extraArgs.
|
||||
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
try {
|
||||
const out = execFileSync('gemini', args, {
|
||||
cwd: opts.workdir,
|
||||
timeout: opts.timeoutMs,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseStreamJson(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login|api key/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse gemini NDJSON stream events:
|
||||
* init → session id (discarded here)
|
||||
* message { delta: true, text } → concat to output
|
||||
* tool_use { name } → increment toolCalls
|
||||
* result { usage: { input_token_count, output_token_count } } → tokens
|
||||
*/
|
||||
private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'message' && typeof obj.text === 'string') {
|
||||
output += obj.text;
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_token_count ?? u.prompt_tokens ?? 0;
|
||||
out += u.output_token_count ?? u.completion_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gemini-2.5-pro',
|
||||
error,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
|
||||
import { estimateCostUsd } from '../pricing';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/**
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
|
||||
*
|
||||
* Codex uses ~/.codex/ for auth (not OPENAI_API_KEY). The --json flag emits
|
||||
* JSONL events; we parse `turn.completed` for usage and `agent_message` / etc.
|
||||
* for output aggregation.
|
||||
*/
|
||||
export class GptAdapter implements ProviderAdapter {
|
||||
readonly name = 'gpt';
|
||||
readonly family = 'gpt' as const;
|
||||
|
||||
async available(): Promise<AvailabilityCheck> {
|
||||
const res = spawnSync('sh', ['-c', 'command -v codex'], { timeout: 2000 });
|
||||
if (res.status !== 0) {
|
||||
return { ok: false, reason: 'codex CLI not found on PATH. Install: npm i -g @openai/codex' };
|
||||
}
|
||||
// Auth sniff: ~/.codex/ should contain auth state after `codex login`
|
||||
const codexDir = path.join(os.homedir(), '.codex');
|
||||
if (!fs.existsSync(codexDir)) {
|
||||
return { ok: false, reason: 'No ~/.codex/ found. Run `codex login` to authenticate via ChatGPT.' };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
const start = Date.now();
|
||||
// `-s read-only` is load-bearing safety. With `--skip-git-repo-check` we
|
||||
// bypass codex's interactive trust prompt for unknown directories (benchmarks
|
||||
// often run in temp dirs / non-git paths), so the read-only sandbox is now
|
||||
// the only boundary preventing codex from mutating the workdir. If you ever
|
||||
// remove `-s read-only`, drop `--skip-git-repo-check` too.
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
|
||||
if (opts.model) args.push('-m', opts.model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
try {
|
||||
const out = execFileSync('codex', args, {
|
||||
cwd: opts.workdir,
|
||||
timeout: opts.timeoutMs,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseJsonl(out);
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gpt-5.4');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse codex exec --json JSONL stream.
|
||||
* Key events:
|
||||
* - item.completed with item.type === 'agent_message' → text output
|
||||
* - item.completed with item.type === 'command_execution' → tool call
|
||||
* - turn.completed → usage.input_tokens, usage.output_tokens
|
||||
* - thread.started → session id (not used here)
|
||||
*/
|
||||
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'item.completed' && obj.item) {
|
||||
if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
|
||||
output += (output ? '\n' : '') + obj.item.text;
|
||||
} else if (obj.item.type === 'command_execution') {
|
||||
toolCalls += 1;
|
||||
}
|
||||
} else if (obj.type === 'turn.completed') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_tokens ?? 0;
|
||||
out += u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines — codex stderr can leak in
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gpt-5.4',
|
||||
error,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
|
||||
*
|
||||
* Each adapter normalizes its provider's result shape into the RunResult below.
|
||||
* The benchmark runner only talks to adapters through this interface.
|
||||
*/
|
||||
|
||||
export interface RunOpts {
|
||||
/** The prompt to send to the model. */
|
||||
prompt: string;
|
||||
/** Working directory passed to the underlying CLI. */
|
||||
workdir: string;
|
||||
/** Hard wall-clock timeout in ms. Default: 300000 (5 min). */
|
||||
timeoutMs: number;
|
||||
/** Specific model within the family, optional. Adapters pass through to provider. */
|
||||
model?: string;
|
||||
/** Extra flags per-provider (escape hatch for rare cases). Prefer staying generic. */
|
||||
extraArgs?: string[];
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
input: number;
|
||||
output: number;
|
||||
/** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */
|
||||
cached?: number;
|
||||
}
|
||||
|
||||
export type RunError =
|
||||
| 'auth' // Credentials missing or invalid.
|
||||
| 'timeout' // Exceeded timeoutMs.
|
||||
| 'rate_limit' // Provider rate-limited us; backoff exceeded.
|
||||
| 'binary_missing' // CLI not found on PATH.
|
||||
| 'unknown'; // Catch-all with reason populated.
|
||||
|
||||
export interface RunResult {
|
||||
/** Provider's textual output for the prompt. */
|
||||
output: string;
|
||||
/** Normalized token usage. 0s if unreported. */
|
||||
tokens: TokenUsage;
|
||||
/** Wall-clock duration. */
|
||||
durationMs: number;
|
||||
/** Count of tool/function calls made during the run (0 if unsupported). */
|
||||
toolCalls: number;
|
||||
/** Actual model ID the provider reports using (may be a variant of the family). */
|
||||
modelUsed: string;
|
||||
/** If the run failed, error code + human reason. output/tokens may be partial. */
|
||||
error?: { code: RunError; reason: string };
|
||||
}
|
||||
|
||||
export interface AvailabilityCheck {
|
||||
ok: boolean;
|
||||
/** When !ok: short reason shown to user. Includes install / login / env var hint. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export type Family = 'claude' | 'gpt' | 'gemini';
|
||||
|
||||
export interface ProviderAdapter {
|
||||
/** Stable name used in output tables and config (e.g., 'claude', 'gpt', 'gemini'). */
|
||||
readonly name: string;
|
||||
/** Model family this adapter targets. */
|
||||
readonly family: Family;
|
||||
/**
|
||||
* Check whether the provider's CLI binary is present and authenticated.
|
||||
* Should never block >2s. Non-throwing: returns { ok: false, reason } on failure.
|
||||
*/
|
||||
available(): Promise<AvailabilityCheck>;
|
||||
/** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
|
||||
run(opts: RunOpts): Promise<RunResult>;
|
||||
/** Estimate USD cost for the reported token usage and model. */
|
||||
estimateCost(tokens: TokenUsage, model?: string): number;
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* Multi-provider benchmark runner.
|
||||
*
|
||||
* Orchestrates running the same prompt across multiple provider adapters and
|
||||
* aggregates RunResult outputs + judge scores into a single report. Adapters
|
||||
* run in parallel (Promise.allSettled) so a slow provider doesn't block a fast
|
||||
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
|
||||
*/
|
||||
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
|
||||
export interface BenchmarkInput {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
timeoutMs?: number;
|
||||
/** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */
|
||||
providers: Array<'claude' | 'gpt' | 'gemini'>;
|
||||
/** Optional per-provider model overrides. */
|
||||
models?: Partial<Record<'claude' | 'gpt' | 'gemini', string>>;
|
||||
/** If true, skip providers whose available() returns !ok. If false, include them with error. */
|
||||
skipUnavailable?: boolean;
|
||||
}
|
||||
|
||||
export interface BenchmarkEntry {
|
||||
provider: string;
|
||||
family: 'claude' | 'gpt' | 'gemini';
|
||||
available: boolean;
|
||||
unavailable_reason?: string;
|
||||
result?: RunResult;
|
||||
costUsd?: number;
|
||||
/** Judge score 0-10 across dimensions. Populated separately by the judge step. */
|
||||
qualityScore?: number;
|
||||
qualityDetails?: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface BenchmarkReport {
|
||||
prompt: string;
|
||||
workdir: string;
|
||||
startedAt: string;
|
||||
durationMs: number;
|
||||
entries: BenchmarkEntry[];
|
||||
}
|
||||
|
||||
const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = {
|
||||
claude: () => new ClaudeAdapter(),
|
||||
gpt: () => new GptAdapter(),
|
||||
gemini: () => new GeminiAdapter(),
|
||||
};
|
||||
|
||||
export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkReport> {
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const timeoutMs = input.timeoutMs ?? 300_000;
|
||||
|
||||
const entries: BenchmarkEntry[] = [];
|
||||
const runPromises: Array<Promise<void>> = [];
|
||||
|
||||
for (const name of input.providers) {
|
||||
const factory = ADAPTERS[name];
|
||||
if (!factory) {
|
||||
entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` });
|
||||
continue;
|
||||
}
|
||||
const adapter = factory();
|
||||
const entry: BenchmarkEntry = { provider: adapter.name, family: adapter.family, available: true };
|
||||
entries.push(entry);
|
||||
|
||||
runPromises.push((async () => {
|
||||
const check = await adapter.available();
|
||||
entry.available = check.ok;
|
||||
if (!check.ok) {
|
||||
entry.unavailable_reason = check.reason;
|
||||
if (input.skipUnavailable) return;
|
||||
}
|
||||
const opts: RunOpts = {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
timeoutMs,
|
||||
model: input.models?.[name],
|
||||
};
|
||||
const res = await adapter.run(opts);
|
||||
entry.result = res;
|
||||
entry.costUsd = adapter.estimateCost(res.tokens, res.modelUsed);
|
||||
})());
|
||||
}
|
||||
|
||||
await Promise.allSettled(runPromises);
|
||||
|
||||
return {
|
||||
prompt: input.prompt,
|
||||
workdir: input.workdir,
|
||||
startedAt,
|
||||
durationMs: Date.now() - startedAtMs,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
|
||||
export function formatJson(report: BenchmarkReport): string {
|
||||
return JSON.stringify(report, null, 2);
|
||||
}
|
||||
|
||||
export function formatMarkdown(report: BenchmarkReport): string {
|
||||
const lines: string[] = [
|
||||
`# Benchmark report — ${report.startedAt}`,
|
||||
'',
|
||||
`**Prompt:** ${report.prompt.length > 200 ? report.prompt.slice(0, 200) + '…' : report.prompt}`,
|
||||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
lines.push(`| ${e.provider} | - | - | - | - | - | unavailable: ${e.unavailable_reason ?? 'unknown'} |`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function msToStr(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function fmtCost(usd?: number): string {
|
||||
if (usd === undefined) return '-';
|
||||
if (usd < 0.01) return `$${usd.toFixed(4)}`;
|
||||
return `$${usd.toFixed(2)}`;
|
||||
}
|
||||
Reference in New Issue
Block a user