gstack/test/benchmark-runner.test.ts

/**
 * Unit tests for the benchmark runner.
 *
 * Mocks adapters to verify:
 * - All adapters run in parallel (Promise.allSettled not serial)
 * - Unavailable adapters are skipped or marked depending on flag
 * - Per-adapter errors don't abort the batch
 * - Output formatters (table, json, markdown) produce non-empty strings
 *
 * Does NOT exercise live CLIs — see test/providers.e2e.test.ts for those.
 */

import { test, expect } from 'bun:test';
import { formatTable, formatJson, formatMarkdown, type BenchmarkReport } from './helpers/benchmark-runner';
import { estimateCostUsd, PRICING } from './helpers/pricing';
import { missingTools, TOOL_COMPATIBILITY } from './helpers/tool-map';
import { CLAUDE_CAPTURE_EVAL_MODEL, CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';

test('estimateCostUsd returns 0 for unknown model (no crash)', () => {
  const cost = estimateCostUsd({ input: 1000, output: 500 }, 'unknown-model-7b');
  expect(cost).toBe(0);
});

test('estimateCostUsd computes correctly for known Claude model', () => {
  // claude-opus-4-7: $5/MTok input, $25/MTok output (platform.claude.com pricing, fetched 2026-10-12)
  // 1M input + 0.5M output = $5 + $12.50 = $17.50
  const cost = estimateCostUsd({ input: 1_000_000, output: 500_000 }, 'claude-opus-4-7');
  expect(cost).toBeCloseTo(17.50, 2);
});

// Opus 5.5 costs $4/MTok input and $20/MTok output; Fable 5.1 (the judge) and Astra $10 and $50.
for (const [model, usd] of [[CLAUDE_CAPTURE_EVAL_MODEL, 0.008], [CLAUDE_JUDGE_EVAL_MODEL, 0.02], [CODEX_FRONTIER_MODEL, 0.02]] as const) {
  test(`estimateCostUsd prices the current ${model} default at its standard rate`, () => {
    expect(estimateCostUsd({ input: 1000, output: 200 }, model)).toBe(usd);
  });
}

test('Fable cache reads use its explicit rate alongside uncached input and output', () => {
  expect(estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-fable-5-1')).toBe(0.25);
  // $0.01 uncached + $0.01 output + $0.001 cache reads, not the legacy 10% rate.
  expect(estimateCostUsd({ input: 1000, output: 200, cached: 4000 }, 'claude-fable-5-1')).toBe(0.021);
});

test('Astra cache reads use its standard cached-input rate', () => {
  expect(estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'gpt-6-astra')).toBe(1);
});

test('estimateCostUsd applies cached input discount alongside uncached input', () => {
  // tokens.input is uncached-only; tokens.cached is disjoint cache-reads at 10%.
  // 0 uncached input, 1M cached → 10% of 5 = $0.50
  const cost1 = estimateCostUsd({ input: 0, output: 0, cached: 1_000_000 }, 'claude-opus-4-7');
  expect(cost1).toBeCloseTo(0.50, 2);
  // 500K uncached input + 500K cached → $2.50 + $0.25 = $2.75
  const cost2 = estimateCostUsd({ input: 500_000, output: 0, cached: 500_000 }, 'claude-opus-4-7');
  expect(cost2).toBeCloseTo(2.75, 2);
});

test('PRICING table covers the key model families', () => {
  expect(PRICING['claude-opus-4-7']).toBeDefined();
  expect(PRICING['claude-sonnet-4-6']).toBeDefined();
  expect(PRICING['gpt-5.4']).toBeDefined();
  expect(PRICING['gemini-2.5-pro']).toBeDefined();
});

test('missingTools reports unsupported tools per provider', () => {
  // GPT/Codex doesn't expose Edit, Glob, Grep
  expect(missingTools('gpt', ['Edit', 'Glob', 'Grep'])).toEqual(['Edit', 'Glob', 'Grep']);
  // Claude supports all core tools
  expect(missingTools('claude', ['Edit', 'Glob', 'Grep', 'Bash', 'Read'])).toEqual([]);
  // Gemini has very limited agentic surface
  expect(missingTools('gemini', ['Bash', 'Edit'])).toEqual(['Bash', 'Edit']);
});

test('TOOL_COMPATIBILITY is populated for all three families', () => {
  expect(TOOL_COMPATIBILITY.claude).toBeDefined();
  expect(TOOL_COMPATIBILITY.gpt).toBeDefined();
  expect(TOOL_COMPATIBILITY.gemini).toBeDefined();
});

test('formatTable handles a report with mixed success/error/unavailable entries', () => {
  const report: BenchmarkReport = {
    prompt: 'test prompt',
    workdir: '/tmp',
    startedAt: '2026-04-16T20:00:00Z',
    durationMs: 1500,
    entries: [
      {
        provider: 'claude',
        family: 'claude',
        available: true,
        result: {
          output: 'ok',
          tokens: { input: 100, output: 200 },
          durationMs: 800,
          toolCalls: 3,
          modelUsed: 'claude-opus-4-7',
        },
        costUsd: 0.0165,
        qualityScore: 9.2,
      },
      {
        provider: 'gpt',
        family: 'gpt',
        available: true,
        result: {
          output: '',
          tokens: { input: 0, output: 0 },
          durationMs: 200,
          toolCalls: 0,
          modelUsed: 'gpt-5.4',
          error: { code: 'auth', reason: 'codex login required' },
        },
      },
      {
        provider: 'gemini',
        family: 'gemini',
        available: false,
        unavailable_reason: 'gemini CLI not on PATH',
      },
    ],
  };

  const table = formatTable(report);
  expect(table).toContain('claude-opus-4-7');
  expect(table).toContain('ERROR auth');
  expect(table).toContain('unavailable');
  expect(table).toContain('9.2/10');
});

test('formatJson produces parseable JSON', () => {
  const report: BenchmarkReport = {
    prompt: 'x',
    workdir: '/tmp',
    startedAt: '2026-04-16T20:00:00Z',
    durationMs: 100,
    entries: [],
  };
  const json = formatJson(report);
  const parsed = JSON.parse(json);
  expect(parsed.prompt).toBe('x');
  expect(parsed.entries).toEqual([]);
});

test('formatMarkdown produces a table header', () => {
  const report: BenchmarkReport = {
    prompt: 'x',
    workdir: '/tmp',
    startedAt: '2026-04-16T20:00:00Z',
    durationMs: 100,
    entries: [],
  };
  const md = formatMarkdown(report);
  expect(md).toContain('# Benchmark report');
  expect(md).toContain('| Model | Latency |');
});