gstack/bin/gstack-model-benchmark

#!/usr/bin/env -S bun --no-env-file --config=/dev/null
/**
 * gstack-model-benchmark — run the same prompt across multiple providers
 * and compare latency, tokens, cost, quality, and tool-call count.
 *
 * Usage:
 *   gstack-model-benchmark <skill-or-prompt-file> [options]
 *
 * Options:
 *   --models claude,gpt,gemini   Comma-separated provider list (default: claude); particle and
 *                                deepseek run in Claude Code through OpenRouter (OPENROUTER_API_KEY)
 *   --prompt "<text>"            Inline prompt instead of a file
 *   --workdir <path>             Working dir passed to each CLI (default: cwd)
 *   --timeout-ms <n>             Per-provider timeout (default: 300000)
 *   --output table|json|markdown Output format (default: table)
 *   --skip-unavailable           Skip providers that fail available() check
 *                                (default: include them with unavailable marker)
 *   --judge                      Run Anthropic SDK judge on outputs for quality score
 *                                (requires ANTHROPIC_API_KEY; adds ~$0.05 per call)
 *   --dry-run                    Validate flags + resolve auth, don't invoke providers
 *
 * Examples:
 *   gstack-model-benchmark --prompt "Write a haiku about databases" --models claude,gpt
 *   gstack-model-benchmark ./test-prompt.txt --models claude,gpt,gemini --judge
 *   gstack-model-benchmark --prompt "hi" --models claude,gpt,gemini --dry-run
 */

import '../lib/conductor-env-shim';
import * as fs from 'fs';
import * as path from 'path';
import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkInput } from '../test/helpers/benchmark-runner';
import { PROVIDERS, PROVIDER_NAMES, isProviderName, type ProviderName } from '../test/helpers/providers/registry';

type OutputFormat = 'table' | 'json' | 'markdown';

const CLI_ARGS = process.argv.slice(2);
const VALUE_FLAGS = new Set(['--models', '--prompt', '--workdir', '--timeout-ms', '--output']);

function arg(name: string, def?: string): string | undefined {
  const idx = CLI_ARGS.findIndex(a => a === name || a.startsWith(name + '='));
  if (idx < 0) return def;
  const eqIdx = CLI_ARGS[idx].indexOf('=');
  if (eqIdx >= 0) return CLI_ARGS[idx].slice(eqIdx + 1);
  return CLI_ARGS[idx + 1];
}

function flag(name: string): boolean {
  return CLI_ARGS.includes(name);
}

function positionalArgs(args: string[]): string[] {
  const positional: string[] = [];
  for (let i = 0; i < args.length; i++) {
    const current = args[i];
    if (current === '--') {
      positional.push(...args.slice(i + 1));
      break;
    }
    if (current.startsWith('--')) {
      const eqIdx = current.indexOf('=');
      const flagName = eqIdx >= 0 ? current.slice(0, eqIdx) : current;
      if (eqIdx < 0 && VALUE_FLAGS.has(flagName) && i + 1 < args.length) {
        i++;
      }
      continue;
    }
    positional.push(current);
  }
  return positional;
}

function parseProviders(s: string | undefined): ProviderName[] {
  if (!s) return ['claude'];
  const seen = new Set<ProviderName>();
  for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) {
    if (isProviderName(p)) seen.add(p);
    else {
      console.error(`WARN: unknown provider '${p}' — skipping. Valid: ${PROVIDER_NAMES.join(', ')}.`);
    }
  }
  if (!seen.size) {
    console.error(`No valid provider in --models '${s}'. Valid: ${PROVIDER_NAMES.join(', ')}.`);
    process.exit(1);
  }
  return Array.from(seen);
}

function parsePositiveIntegerFlag(name: string, def: string): number {
  const raw = arg(name, def);
  if (!raw || !/^\+?[1-9]\d*$/.test(raw)) {
    console.error(`${name} requires a positive integer`);
    process.exit(1);
  }
  const parsed = Number(raw);
  if (!Number.isSafeInteger(parsed)) {
    console.error(`${name} requires a positive integer`);
    process.exit(1);
  }
  return parsed;
}

function resolvePrompt(positional: string | undefined): string {
  const inline = arg('--prompt');
  if (inline) return inline;
  if (!positional) {
    console.error('ERROR: specify a prompt via positional path or --prompt "<text>"');
    process.exit(1);
  }
  if (fs.existsSync(positional)) {
    return fs.readFileSync(positional, 'utf-8');
  }
  // Not a file — treat as inline prompt
  return positional;
}

async function main(): Promise<void> {
  const positional = positionalArgs(CLI_ARGS)[0];
  const prompt = resolvePrompt(positional);
  const providers = parseProviders(arg('--models'));
  const workdir = arg('--workdir', process.cwd())!;
  const timeoutMs = parsePositiveIntegerFlag('--timeout-ms', '300000');
  const output = (arg('--output', 'table') as OutputFormat);
  const skipUnavailable = flag('--skip-unavailable');
  const doJudge = flag('--judge');
  const dryRun = flag('--dry-run');

  if (dryRun) {
    await dryRunReport({ prompt, providers, workdir, timeoutMs, output, doJudge });
    return;
  }

  const input: BenchmarkInput = {
    prompt,
    workdir,
    providers,
    timeoutMs,
    skipUnavailable,
  };

  const report = await runBenchmark(input);

  if (doJudge) {
    try {
      const { judgeEntries } = await import('../test/helpers/benchmark-judge');
      await judgeEntries(report);
    } catch (err) {
      console.error(`WARN: judge unavailable: ${(err as Error).message}`);
    }
  }

  let out: string;
  switch (output) {
    case 'json':     out = formatJson(report); break;
    case 'markdown': out = formatMarkdown(report); break;
    case 'table':
    default:         out = formatTable(report); break;
  }
  process.stdout.write(out + '\n');
}

async function dryRunReport(opts: {
  prompt: string;
  providers: ProviderName[];
  workdir: string;
  timeoutMs: number;
  output: OutputFormat;
  doJudge: boolean;
}): Promise<void> {
  const lines: string[] = [];
  lines.push('== gstack-model-benchmark --dry-run ==');
  lines.push(`  prompt:     ${opts.prompt.length > 80 ? opts.prompt.slice(0, 80) + '…' : opts.prompt}`);
  lines.push(`  providers:  ${opts.providers.join(', ')}`);
  lines.push(`  workdir:    ${opts.workdir}`);
  lines.push(`  timeout_ms: ${opts.timeoutMs}`);
  lines.push(`  output:     ${opts.output}`);
  lines.push(`  judge:      ${opts.doJudge ? 'on (Anthropic SDK)' : 'off'}`);
  lines.push('');
  lines.push('Adapter availability:');
  let authFailures = 0;
  for (const name of opts.providers) {
    const factory = PROVIDERS[name];
    if (!factory) {
      lines.push(`  ${name}: UNKNOWN PROVIDER`);
      authFailures += 1;
      continue;
    }
    const adapter = factory();
    const check = await adapter.available();
    if (check.ok) {
      lines.push(`  ${adapter.name}: OK`);
    } else {
      lines.push(`  ${adapter.name}: NOT READY — ${check.reason}`);
      authFailures += 1;
    }
  }
  lines.push('');
  lines.push(`(--dry-run — no prompts sent. ${authFailures} provider(s) unavailable.)`);
  process.stdout.write(lines.join('\n') + '\n');
}

main().catch(err => {
  console.error('FATAL:', err);
  process.exit(1);
});