gstack/bin/gstack-model-benchmark

#!/usr/bin/env bun
/**
 * gstack-model-benchmark — run the same prompt(s) across providers and compare,
 * with scoring, experiments, and reporting owned by Braintrust (local by default;
 * nothing is uploaded unless BRAINTRUST_API_KEY is set — that key is the consent
 * boundary for the cloud dashboard).
 *
 * Usage:
 *   gstack-model-benchmark [<prompt-file>] [options]
 *
 * Options:
 *   --models claude,gpt,gemini   Comma-separated provider list (default: claude)
 *   --prompt "<text>"            Ad-hoc single-case prompt instead of the corpus
 *   --corpus <path>              Corpus JSON (default: evals/model-benchmark/corpus.json)
 *   --timeout-ms <n>             Per-provider-per-case timeout (default: 300000)
 *   --output table|json          Output format (default: table)
 *   --judge                      Add the autoevals ClosedQA LLM judge (needs OPENAI_API_KEY)
 *   --dry-run                    Validate flags + resolve auth, don't invoke providers
 *
 * Examples:
 *   gstack-model-benchmark --prompt "Write a haiku about databases" --models claude,gpt
 *   gstack-model-benchmark --models claude,gpt,gemini            # runs the corpus
 *   gstack-model-benchmark --prompt "hi" --models claude,gpt,gemini --dry-run
 */

import '../lib/conductor-env-shim';
import * as fs from 'fs';
import {
  loadCorpus,
  runProviderBenchmark,
  type BenchCase,
  type ProviderBenchmark,
  type ProviderName,
} from '../lib/model-benchmark/braintrust-eval';
import { ClaudeAdapter } from '../lib/model-benchmark/providers/claude';
import { GptAdapter } from '../lib/model-benchmark/providers/gpt';
import { GeminiAdapter } from '../lib/model-benchmark/providers/gemini';

const ADAPTER_FACTORIES = {
  claude: () => new ClaudeAdapter(),
  gpt: () => new GptAdapter(),
  gemini: () => new GeminiAdapter(),
};

type OutputFormat = 'table' | 'json';

const CLI_ARGS = process.argv.slice(2);
const VALUE_FLAGS = new Set(['--models', '--prompt', '--corpus', '--timeout-ms', '--output']);

function arg(name: string, def?: string): string | undefined {
  const idx = CLI_ARGS.findIndex(a => a === name || a.startsWith(name + '='));
  if (idx < 0) return def;
  const eqIdx = CLI_ARGS[idx].indexOf('=');
  if (eqIdx >= 0) return CLI_ARGS[idx].slice(eqIdx + 1);
  return CLI_ARGS[idx + 1];
}

function flag(name: string): boolean {
  return CLI_ARGS.includes(name);
}

function positionalArgs(args: string[]): string[] {
  const positional: string[] = [];
  for (let i = 0; i < args.length; i++) {
    const current = args[i];
    if (current === '--') {
      positional.push(...args.slice(i + 1));
      break;
    }
    if (current.startsWith('--')) {
      const eqIdx = current.indexOf('=');
      const flagName = eqIdx >= 0 ? current.slice(0, eqIdx) : current;
      if (eqIdx < 0 && VALUE_FLAGS.has(flagName) && i + 1 < args.length) {
        i++;
      }
      continue;
    }
    positional.push(current);
  }
  return positional;
}

function parseProviders(s: string | undefined): ProviderName[] {
  if (!s) return ['claude'];
  const seen = new Set<ProviderName>();
  for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) {
    if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p);
    else console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`);
  }
  return seen.size ? Array.from(seen) : ['claude'];
}

/** Resolve the cases: --prompt/file → single ad-hoc case; else the corpus. */
function resolveCases(positional: string | undefined): BenchCase[] {
  const inline = arg('--prompt');
  if (inline) return [{ id: 'adhoc', input: inline, required: [] }];
  if (positional && fs.existsSync(positional)) {
    return [{ id: 'adhoc', input: fs.readFileSync(positional, 'utf-8'), required: [] }];
  }
  if (positional) return [{ id: 'adhoc', input: positional, required: [] }];
  return loadCorpus(arg('--corpus'));
}

async function main(): Promise<void> {
  const positional = positionalArgs(CLI_ARGS)[0];
  const providers = parseProviders(arg('--models'));
  const timeoutMs = parseInt(arg('--timeout-ms', '300000')!, 10);
  const output = (arg('--output', 'table') as OutputFormat);
  const doJudge = flag('--judge');
  const dryRun = flag('--dry-run');

  if (dryRun) {
    await dryRunReport({ cases: resolveCases(positional), providers, timeoutMs, output, doJudge });
    return;
  }

  if (doJudge && !process.env.OPENAI_API_KEY) {
    console.error('WARN: --judge needs OPENAI_API_KEY for the autoevals ClosedQA judge — running with the deterministic scorer only.');
  }

  const cases = resolveCases(positional);
  const results: ProviderBenchmark[] = [];
  for (const provider of providers) {
    results.push(await runProviderBenchmark(provider, cases, { timeoutMs, judge: doJudge && !!process.env.OPENAI_API_KEY }));
  }

  process.stdout.write((output === 'json' ? JSON.stringify(results, null, 2) : formatTable(results)) + '\n');
}

function formatTable(results: ProviderBenchmark[]): string {
  const header = `Provider   Score    Cases  Latency(avg)  Errors`;
  const rows: string[] = [header, '-'.repeat(header.length)];
  for (const r of results) {
    const n = r.ops.length || 1;
    const avgMs = r.ops.reduce((s, o) => s + o.durationMs, 0) / n;
    const errs = r.ops.filter(o => o.error).length;
    const score = r.score === null ? '—' : `${(r.score * 100).toFixed(1)}%`;
    rows.push(
      `${pad(r.provider, 10)} ${pad(score, 8)} ${pad(String(r.ops.length), 6)} ${pad(msToStr(avgMs), 13)} ${errs || ''}`,
    );
  }
  return rows.join('\n');
}

async function dryRunReport(opts: {
  cases: BenchCase[];
  providers: ProviderName[];
  timeoutMs: number;
  output: OutputFormat;
  doJudge: boolean;
}): Promise<void> {
  const adhoc = opts.cases.length === 1 && opts.cases[0].id === 'adhoc';
  const lines: string[] = [];
  lines.push('== gstack-model-benchmark --dry-run ==');
  if (adhoc) {
    const p = opts.cases[0].input;
    lines.push(`  prompt:     ${p.length > 80 ? p.slice(0, 80) + '…' : p}`);
  } else {
    lines.push(`  corpus:     ${opts.cases.length} case(s)`);
  }
  lines.push(`  providers:  ${opts.providers.join(', ')}`);
  lines.push(`  timeout_ms: ${opts.timeoutMs}`);
  lines.push(`  output:     ${opts.output}`);
  lines.push(`  judge:      ${opts.doJudge ? 'on (autoevals ClosedQA)' : 'off'}`);
  lines.push(`  upload:     ${process.env.BRAINTRUST_API_KEY ? 'ON — BRAINTRUST_API_KEY set (cloud dashboard)' : 'off (local only, nothing uploaded)'}`);
  lines.push('');
  lines.push('Adapter availability:');
  let authFailures = 0;
  for (const name of opts.providers) {
    const factory = ADAPTER_FACTORIES[name];
    if (!factory) {
      lines.push(`  ${name}: UNKNOWN PROVIDER`);
      authFailures += 1;
      continue;
    }
    const check = await factory().available();
    if (check.ok) lines.push(`  ${name}: OK`);
    else { lines.push(`  ${name}: NOT READY — ${check.reason}`); authFailures += 1; }
  }
  lines.push('');
  lines.push(`(--dry-run — no prompts sent. ${authFailures} provider(s) unavailable.)`);
  process.stdout.write(lines.join('\n') + '\n');
}

function pad(s: string, n: number): string {
  return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
}
function msToStr(ms: number): string {
  if (ms < 1000) return `${Math.round(ms)}ms`;
  return `${(ms / 1000).toFixed(1)}s`;
}

main().catch(err => {
  console.error('FATAL:', err);
  process.exit(1);
});