gstack/test/pricing-eval-plan.test.ts

/**
 * PR E1: lib/pricing.ts (the table test/helpers/pricing.ts re-exports) and the
 * /eval-plan machinery in lib/eval-plan.ts: preregistration check, priced arms,
 * the $0 dry run, and the priced pilot ledger with its spend table.
 */
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { EXCLUDED_BILLING_CATEGORIES, PRICING, estimateCostUsd, hasPricing, pricingRow } from '../lib/pricing';
import * as helperPricing from './helpers/pricing';
import {
  PREREG_SECTIONS, checkPrereg, dryRun, formatEvalCalls, parseEvalCalls, pilotReport, preregTemplate, renderArms, renderDryRun, renderPilotReport, renderSpendTable,
  resolveArms, settlePilot, startPilot, estimateTokens, REASONING_OUTPUT_FLOOR, isSmallTierModel,
} from '../lib/eval-plan';
import { capyHost, type CapyHost } from './helpers/capy-host';

const dirs: string[] = [];
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-plan-')); dirs.push(d); return d; };
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });

describe('lib/pricing', () => {
  test('the helper re-exports the lib table unchanged and documents the excluded billing categories', () => {
    expect(helperPricing.PRICING).toBe(PRICING);
    expect(helperPricing.estimateCostUsd).toBe(estimateCostUsd);
    expect(EXCLUDED_BILLING_CATEGORIES.length).toBeGreaterThan(0);
    for (const row of Object.values(PRICING)) { expect(row.input_per_mtok).toBeGreaterThan(0); expect(row.output_per_mtok).toBeGreaterThan(0); expect(row.as_of).toMatch(/^\d{4}-\d{2}$/); }
  });

  test('provider-prefixed ids resolve to their row; unknown models price at 0 and report missing', () => {
    expect(pricingRow('openai/gpt-6-astra')).toBe(PRICING['gpt-6-astra']);
    expect(hasPricing('anthropic/claude-fable-5-1')).toBe(true);
    expect(hasPricing('nobody/made-this-up')).toBe(false);
    expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'gpt-6-astra')).toBe(60);
    expect(estimateCostUsd({ input: 1_000_000, output: 0, cached: 1_000_000 }, 'claude-fable-5-1')).toBe(10.25);
    expect(estimateCostUsd({ input: 10, output: 10 }, 'nobody/made-this-up')).toBe(0);
  });

  test('the default eval-arms tier (smart) is priced out of the box', () => {
    for (const model of ['claude-opus-5-5', 'gpt-6.1-sol']) expect(hasPricing(model)).toBe(true);
  });
});

describe('preregistration', () => {
  test('the template has every required section and an empty section fails the check', () => {
    const t = preregTemplate('x');
    const check = checkPrereg(t);
    expect(check.missing).toEqual([]);
    expect(check.empty.length).toBe(PREREG_SECTIONS.length);
    expect(check.ok).toBe(false);
    expect(PREREG_SECTIONS).toContain('Comparison');
    const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack');
    expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] });
    expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]);
  });
});

describe('arms', () => {
  test('explicit arms are priced from the table; an unpriced arm is EVAL_PRICE_MISSING', () => {
    const d = tmp();
    const priced = resolveArms({ cwd: d, explicit: ['gpt-6-astra', 'claude-fable-5-1'] });
    expect(priced.errors).toEqual([]);
    expect(priced.arms.map(a => a.priced)).toEqual([true, true]);
    expect(renderArms(priced, { input: 1000, output: 1000 })).toContain('ARM: gpt-6-astra');
    const unpriced = resolveArms({ cwd: d, explicit: ['gpt-6-astra', 'made-up-model'] });
    expect(unpriced.errors.map(e => e.code)).toEqual(['EVAL_PRICE_MISSING']);
    expect(renderArms(unpriced, { input: 1000, output: 1000 })).toContain('price=MISSING');
  });

  test('the eval-arms role is read through the models CLI seam, never the network', () => {
    const d = tmp();
    const calls: string[][] = [];
    const selections = [
      { provider: 'openai', requestedModel: 'gpt-6-astra', status: 'selected', tier: 'implementation' },
      { provider: 'anthropic', requestedModel: 'claude-fable-5-1', status: 'selected', tier: 'implementation' },
      { provider: 'google', requestedModel: 'gemini-x', status: 'unavailable' },
    ];
    const r = resolveArms({ cwd: d, modelsCli: args => { calls.push(args); return { code: 0, stdout: JSON.stringify({ selections }) }; } });
    expect(calls).toEqual([['resolve', '--role', 'eval-arms', '--json']]);
    expect(r.source).toBe('policy:eval-arms');
    expect(r.arms.map(a => [a.model, a.provider, a.source])).toEqual([
      ['gpt-6-astra', 'openai', 'policy:eval-arms/implementation'],
      ['claude-fable-5-1', 'anthropic', 'policy:eval-arms/implementation'],
    ]);
    const none = resolveArms({ cwd: d, modelsCli: () => ({ code: 1, stdout: '' }) });
    expect(none.errors.map(e => e.code)).toEqual(['EVAL_ARMS_UNRESOLVED']);
  });
});

describe('dry run', () => {
  test('flags malformed rows and missing fields, requires the retry prompt and the output floor, and prices per item', () => {
    const d = tmp();
    const rows = path.join(d, 'rows.jsonl');
    fs.writeFileSync(rows, [JSON.stringify({ id: 1, prompt: 'hello world' }), 'not json', JSON.stringify({ id: 3 }), ''].join('\n'));
    const retry = path.join(d, 'retry.txt');
    fs.writeFileSync(retry, 'Try again with the schema.\n');
    const r = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry });
    expect(r.rows).toBe(3);
    expect(r.bad_rows.map(b => b.line)).toEqual([2, 3]);
    expect(r.bad_rows[0]!.missing).toEqual(['<not a JSON object>']);
    expect(r.bad_rows[1]!.missing).toEqual(['prompt']);
    expect(r.pass).toBe(false);
    expect(r.per_item_usd['gpt-6-astra']).toBeGreaterThan(0);
    expect(renderDryRun(r)).toContain('DRY_RUN:');
    fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello world' }) + '\n');
    expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry }).pass).toBe(true);
    expect(dryRun({ input: rows, arms: ['gpt-6-astra'] }).checks.retry_prompt.ok).toBe(false);
    expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, outputBudget: 100 }).checks.output_budget.ok).toBe(false);
    expect(dryRun({ input: rows, arms: ['made-up'], retryPrompt: retry }).checks.price_table.missing).toEqual(['made-up']);
  });

  test('R54: refuses a Haiku- or Sonnet-only plan unless it names a tier question, warns on a mix, and re-costs on frontier prices', () => {
    const d = tmp();
    const rows = path.join(d, 'rows.jsonl');
    fs.writeFileSync(rows, [JSON.stringify({ prompt: 'hello world' }), JSON.stringify({ prompt: 'second row' })].join('\n') + '\n');
    const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
    const small = dryRun({ input: rows, arms: ['claude-haiku-4-5', 'claude-sonnet-5-5#no-gstack'], retryPrompt: retry });
    expect(small.pass).toBe(false);
    expect(small.checks.frontier).toMatchObject({ ok: false, small_tier: ['claude-haiku-4-5', 'claude-sonnet-5-5#no-gstack'], tier_question: null });
    const avgIn = Math.ceil((estimateTokens('hello world') + estimateTokens('second row')) / 2);
    expect(small.checks.frontier.recost_usd['claude-opus-5-5']).toBe(2 * 2 * estimateCostUsd({ input: avgIn, output: REASONING_OUTPUT_FLOOR }, 'claude-opus-5-5'));
    expect(small.checks.frontier.recost_usd['gpt-6.1-sol']).toBeGreaterThan(0);
    expect(renderDryRun(small)).toContain('CHECK: frontier refused');
    const asked = dryRun({ input: rows, arms: ['claude-haiku-4-5'], retryPrompt: retry, tierQuestion: 'does the diet hold on Haiku' });
    expect(asked.pass).toBe(true);
    expect(renderDryRun(asked)).toContain('CHECK: frontier ok (tier question');
    const mixed = dryRun({ input: rows, arms: ['claude-opus-5-5', 'claude-sonnet-5-5'], retryPrompt: retry });
    expect(mixed.pass).toBe(true);
    expect(renderDryRun(mixed)).toContain('CHECK: frontier warn');
    const frontier = dryRun({ input: rows, arms: ['claude-opus-5-5', 'gpt-6.1-sol#no-gstack'], retryPrompt: retry });
    expect(frontier.checks.frontier).toMatchObject({ ok: true, small_tier: [] });
    expect(renderDryRun(frontier)).not.toContain('CHECK: frontier');
    expect(['claude-opus-5-5', 'gpt-6.1-sol', 'gpt-6-luna', 'gemini-3.8-flash'].some(isSmallTierModel)).toBe(false);
    const dryFile = (name: string, r: unknown) => { const f = path.join(d, name); fs.writeFileSync(f, JSON.stringify(r)); return f; };
    const swapped = startPilot({ out: path.join(d, 'swapped'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('frontier.json', frontier) });
    expect(swapped.error?.code).toBe('EVAL_DRY_RUN_FAILED');
    expect(fs.existsSync(path.join(d, 'swapped'))).toBe(false);
    expect(startPilot({ out: path.join(d, 'asked'), cap: 5, n: 1, arms: ['claude-haiku-4-5'], dryRun: dryFile('asked.json', asked) }).error).toBeUndefined();
    expect(startPilot({ out: path.join(d, 'mixed'), cap: 5, n: 1, arms: ['claude-opus-5-5', 'claude-sonnet-5-5'], dryRun: dryFile('frontier2.json', frontier) }).error).toBeUndefined();
  });

  test('R54 CLI: --tier-question refuses another flag as its answer', () => {
    const host = capyHost();
    try {
      const rows = path.join(host.base, 'rows.jsonl'); fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello world' }) + '\n');
      const retry = path.join(host.base, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
      const run = (...extra: string[]) => spawnSync(BIN, ['dry-run', '--input', rows, '--arms', 'claude-haiku-4-5', '--retry-prompt', retry, ...extra], { cwd: host.base, env: host.env, encoding: 'utf8', timeout: 30_000 });
      const flagAnswer = run('--tier-question', '--json');
      expect(flagAnswer.status).not.toBe(0);
      expect(flagAnswer.stderr).toContain('--tier-question needs the question');
      const refused = run();
      expect(refused.status).not.toBe(0);
      expect(refused.stderr).toContain('R54');
      expect(run('--tier-question', 'does the diet hold on Haiku').status).toBe(0);
    } finally { host.cleanup(); }
  });
});

describe('priced pilot', () => {
  function passingDryRun(d: string): string {
    const rows = path.join(d, 'rows.jsonl');
    fs.writeFileSync(rows, JSON.stringify({ prompt: 'x'.repeat(400) }) + '\n');
    const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
    const file = path.join(d, 'dry-run.json');
    fs.writeFileSync(file, JSON.stringify(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry })));
    return file;
  }

  test('reserves n attempts per arm under the cap, settles actual spend, and projects the full run with a band', () => {
    const d = tmp();
    const out = path.join(d, 'run');
    const started = startPilot({ out, cap: 10, n: 3, arms: ['gpt-6-astra'], dryRun: passingDryRun(d), now: new Date('2026-10-10T00:00:00Z') });
    expect(started.error).toBeUndefined();
    expect(started.record.attempts).toHaveLength(3);
    expect(started.record.status).toBe('reserved');
    for (const [i, a] of started.record.attempts.entries()) settlePilot(out, a.attempt, 0.4 + i * 0.01);
    const report = pilotReport(out, { total: 20, workers: 2 });
    expect(report.settled_known).toBe(3);
    expect(report.projection.per_item_usd).toBeCloseTo(0.41, 6);
    expect(report.projection.projected_usd).toBeCloseTo(8.2, 6);
    expect(report.projection.band_usd![0]).toBeLessThan(report.projection.projected_usd!);
    expect(report.projection.verdict).toBe('within');
    const text = renderPilotReport(report);
    expect(text).toContain('PILOT:');
    expect(renderSpendTable(report.spend, 10)).toContain('cap');
  });

  test('a reservation past the cap ends the pilot EVAL_BUDGET_EXHAUSTED; a failed dry run never starts it; unknown spend keeps the verdict unknown', () => {
    const d = tmp();
    const dry = passingDryRun(d);
    const exhausted = startPilot({ out: path.join(d, 'tight'), cap: 0.0001, n: 3, arms: ['gpt-6-astra'], dryRun: dry });
    expect(exhausted.error?.code).toBe('EVAL_BUDGET_EXHAUSTED');
    expect(exhausted.record.status).toBe('budget_exhausted');
    const failed = path.join(d, 'failed.json');
    fs.writeFileSync(failed, JSON.stringify({ ...JSON.parse(fs.readFileSync(dry, 'utf8')), pass: false }));
    expect(startPilot({ out: path.join(d, 'never'), cap: 10, n: 3, arms: ['gpt-6-astra'], dryRun: failed }).error?.code).toBe('EVAL_DRY_RUN_FAILED');
    expect(fs.existsSync(path.join(d, 'never', 'pilot.json'))).toBe(false);
    const out = path.join(d, 'unknown');
    const s = startPilot({ out, cap: 10, n: 2, arms: ['gpt-6-astra'], dryRun: dry });
    settlePilot(out, s.record.attempts[0]!.attempt, 0.3);
    settlePilot(out, s.record.attempts[1]!.attempt, 'unknown');
    const r = pilotReport(out, { total: 5 });
    expect(r.settled_unknown).toBe(1);
    expect(r.projection.verdict).toBe('unknown');
  });

  test('#3653: each arm is projected on its own, so arms at different prices that fit the cap are not told they exceed it', () => {
    const d = tmp();
    const out = path.join(d, 'mixed');
    const s = startPilot({ out, cap: 2, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
    expect(s.error).toBeUndefined();
    const usd: Record<string, number[]> = { 'gpt-6-astra': [0.001, 0.001, 0.001], 'gpt-5.4-mini': [0.10, 0.11, 0.12] };
    for (const a of s.record.attempts) settlePilot(out, a.attempt, usd[a.model]![a.row - 1]!);
    const r = pilotReport(out, { total: 10, workers: 1 });
    expect(r.projection.projected_usd).toBeCloseTo(1.11, 6);
    expect(r.projection.band_usd![0]).toBeCloseTo(1.11 - 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
    expect(r.projection.band_usd![1]).toBeCloseTo(1.11 + 4.303 * 0.01 / Math.sqrt(3) * 10, 4);
    expect(r.projection.verdict).toBe('within');
    expect(r.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 3], ['gpt-5.4-mini', 3]]);
    expect(renderPilotReport(r)).toContain('PILOT_ARM: arm=gpt-5.4-mini n=3 per_item=$0.1100');
  });

  test('#3653: a priced arm mean is never multiplied across an arm settled unknown', () => {
    const d = tmp();
    const out = path.join(d, 'partial');
    const s = startPilot({ out, cap: 10, n: 3, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: passingDryRun(d) });
    for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 0.0003 : 'unknown');
    const r = pilotReport(out, { total: 400 });
    expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
    expect(r.projection.verdict).toBe('unknown');
    expect(r.settled_unknown).toBe(3);
    const text = renderPilotReport(r);
    expect(text).toContain('known_arms=1/2');
    expect(text).toContain('--seat-metered');
  });

  test('#3653: a declared seat-metered arm holds none of the cap, stays out of the verdict and gets its own line', () => {
    const d = tmp();
    const dry = passingDryRun(d);
    const est = startPilot({ out: path.join(d, 'probe'), cap: 100, n: 1, arms: ['gpt-6-astra', 'gpt-5.4-mini'], dryRun: dry }).record.attempts;
    const astra = est.find(a => a.model === 'gpt-6-astra')!.estimate_usd, mini = est.find(a => a.model === 'gpt-5.4-mini')!.estimate_usd;
    const cap = Number((3 * mini + 2 * astra).toFixed(4));
    expect(startPilot({ out: path.join(d, 'undeclared'), cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], dryRun: dry }).error?.code).toBe('EVAL_BUDGET_EXHAUSTED');
    const out = path.join(d, 'seat');
    const s = startPilot({ out, cap, n: 3, arms: ['gpt-5.4-mini', 'gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
    expect(s.error).toBeUndefined();
    expect(s.record.seat_metered).toEqual(['gpt-6-astra']);
    expect(s.record.attempts.filter(a => a.model === 'gpt-6-astra').map(a => a.estimate_usd)).toEqual([0, 0, 0]);
    for (const a of s.record.attempts) settlePilot(out, a.attempt, a.model === 'gpt-6-astra' ? 'unknown' : 0.0003);
    const r = pilotReport(out, { total: 400, workers: 3 });
    expect(r.settled_unknown).toBe(0);
    expect(r.spend.summary.unknown_usd).toBe(0);
    expect(r.spend.max_estimate_usd).toBeCloseTo(mini, 6);
    expect(r.projection.projected_usd).toBeCloseTo(0.12, 6);
    expect(r.projection.verdict).toBe('within');
    expect(r.seat_metered).toEqual([{ arm: 'gpt-6-astra', calls: 3, settled: 3 }]);
    expect(renderPilotReport(r)).toContain('PILOT_SEAT_METERED: arm=gpt-6-astra calls=3 settled=3');
  });

  test('#3653: a charge on a seat-metered attempt still counts as spent; the declaration names listed arms and is fixed for its --out', () => {
    const d = tmp();
    const dry = passingDryRun(d);
    const out = path.join(d, 'seat');
    const s = startPilot({ out, cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
    settlePilot(out, s.record.attempts[0]!.attempt, 0.05);
    const charged = pilotReport(out, { total: 1000 });
    expect(charged.spend.summary.spent_usd).toBeCloseTo(0.05, 6);
    expect(charged.seat_metered).toEqual([]);
    expect(charged.projection.arms.map(a => [a.arm, a.n])).toEqual([['gpt-6-astra', 1]]);
    expect(charged.projection.verdict).toBe('unknown');
    expect(() => startPilot({ out: path.join(d, 'stray'), cap: 10, n: 1, arms: ['gpt-6-astra'], seatMetered: ['gpt-5.4-mini'], dryRun: dry })).toThrow('--seat-metered');
    const switched = path.join(d, 'switched');
    startPilot({ out: switched, cap: 1, n: 3, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
    expect(() => startPilot({ out: switched, cap: 1, n: 3, arms: ['gpt-6-astra'], dryRun: dry })).toThrow('needs a new --out');
    expect(JSON.parse(fs.readFileSync(path.join(switched, 'pilot.json'), 'utf8')).seat_metered).toEqual(['gpt-6-astra']);
  });

  test('#3653: an all-seat-metered pilot spends nothing and fits; one priced attempt is not enough for a verdict', () => {
    const d = tmp();
    const dry = passingDryRun(d);
    const seat = path.join(d, 'all-seat');
    const s = startPilot({ out: seat, cap: 1, n: 3, arms: ['gpt-6-astra'], seatMetered: ['gpt-6-astra'], dryRun: dry });
    for (const a of s.record.attempts) settlePilot(seat, a.attempt, 'unknown');
    const all = pilotReport(seat, { total: 50 });
    expect([all.projection.projected_usd, all.projection.verdict]).toEqual([0, 'within']);
    expect(renderPilotReport(all)).toContain('known_arms=0/0');
    const one = path.join(d, 'one');
    settlePilot(one, startPilot({ out: one, cap: 10, n: 1, arms: ['gpt-6-astra'], dryRun: dry }).record.attempts[0]!.attempt, 0.1);
    const r = pilotReport(one, { total: 50 });
    expect([r.projection.band_usd, r.projection.verdict]).toEqual([null, 'unknown']);
    expect(renderPilotReport(r)).toContain('settle at least 2 attempts per priced arm');
  });

  test('#3653: a seat-metered arm still needs a price row, as the dry run does', () => {
    const d = tmp();
    const s = startPilot({ out: path.join(d, 'seat'), cap: 1, n: 2, arms: ['gpt-5.4-mini', 'seat-only-model'], seatMetered: ['seat-only-model'], dryRun: passingDryRun(d) });
    expect(s.error?.code).toBe('EVAL_PRICE_MISSING');
  });

  test('#3653: pilot start --seat-metered reaches the ledger through the CLI', () => {
    const d = tmp();
    const out = path.join(d, 'cli');
    const r = spawnSync(BIN, ['pilot', 'start', '--out', out, '--cap', '10', '--arms', 'gpt-5.4-mini,gpt-6-astra', '--seat-metered', 'gpt-6-astra', '--dry-run', passingDryRun(d)], { cwd: d, encoding: 'utf8', timeout: 30_000 });
    expect(r.status).toBe(0);
    expect(r.stdout).toContain('PILOT_ATTEMPT: pilot-gpt-6-astra-1 model=gpt-6-astra estimate=$0.0000');
    expect(JSON.parse(fs.readFileSync(path.join(out, 'pilot.json'), 'utf8')).seat_metered).toEqual(['gpt-6-astra']);
  });
});

// ---------------------------------------------------------------------------
// CD2 (field wave item 15): paid gates priced from a call-plan DATA file.
// `price-gate` never executes anything: it validates the file's binding (gate
// command, head, input hashes at HEAD) and prices the GSTACK_EVAL_CALLS v1 rows.
// ---------------------------------------------------------------------------
const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-eval-plan');

function scratchRepo(env: Record<string, string>): { repo: string; head: string; rowsSha: string } {
  const repo = path.join(env.TMPDIR!, 'repo');
  fs.mkdirSync(repo, { recursive: true });
  const git = (...args: string[]) => {
    const r = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30_000 });
    if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
    return r.stdout.trim();
  };
  git('init', '-q', '-b', 'main');
  fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n');
  git('add', 'rows.jsonl');
  git('commit', '-q', '-m', 'rows');
  return { repo, head: git('rev-parse', 'HEAD'), rowsSha: createHash('sha256').update(fs.readFileSync(path.join(repo, 'rows.jsonl'))).digest('hex') };
}

const GATE = 'bun run eval:synopsis --questions 50 --arms opus,sol';
const twoRows = [
  { purpose: 'answer', model: 'anthropic/claude-opus-5-5', calls: 100, in_tokens: 400_000, out_tokens: 200_000, max_out_tokens: 8_192, scope: 'full' as const },
  { purpose: 'judge', model: 'gpt-6.1-sol', calls: 100, in_tokens: 300_000, out_tokens: 50_000, max_out_tokens: 2_048, scope: 'full' as const },
];

function runGate(host: CapyHost, cwd: string, args: string[], extraEnv: Record<string, string> = {}) {
  const r = spawnSync(BIN, ['price-gate', ...args], { cwd, env: { ...host.env, ...extraEnv }, encoding: 'utf8', timeout: 30_000 });
  return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
}

describe('price-gate: the call-plan file format (formatEvalCalls / parseEvalCalls)', () => {
  test('the reference emitter round-trips: binding lines, preconditions, call lines and the end line', () => {
    const plan = { gate: GATE, head: 'a'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: 'b'.repeat(64) }], preconditions: ['ANTHROPIC_API_KEY set at run time'], rows: twoRows };
    const text = formatEvalCalls(plan);
    expect(text).toContain(`GSTACK_EVAL_GATE v1: ${GATE}\n`);
    expect(text).toContain('GSTACK_EVAL_CALLS v1: purpose=answer model=anthropic/claude-opus-5-5 calls=100 in_tokens=400000 out_tokens=200000 max_out_tokens=8192 scope=full\n');
    expect(text.trimEnd().split('\n').at(-1)).toBe('GSTACK_EVAL_CALLS v1 end: lines=2');
    const parsed = parseEvalCalls(text);
    expect('plan' in parsed && parsed.plan).toEqual(plan);
  });

  test('a malformed, partial or unknown-version file refuses and names the line', () => {
    const good = formatEvalCalls({ gate: GATE, head: 'a'.repeat(40), inputs: [], preconditions: [], rows: twoRows });
    const err = (text: string) => { const r = parseEvalCalls(text); return 'error' in r ? r.error : ''; };
    expect(err(good.replace(/\nGSTACK_EVAL_CALLS v1 end: lines=2\n$/, '\n'))).toMatch(/end line/);
    expect(err(good.replace('end: lines=2', 'end: lines=3'))).toMatch(/lines=3.*2 call lines/);
    expect(err(good.replace('calls=100 in_tokens=400000', 'calls=1e2 in_tokens=400000'))).toMatch(/line 4.*calls/);
    expect(err(good.replace('scope=full\nGSTACK_EVAL_CALLS v1: purpose=judge', 'scope=full extra=1\nGSTACK_EVAL_CALLS v1: purpose=judge'))).toMatch(/line 4.*extra/);
    expect(err(good.replace(' max_out_tokens=8192', ''))).toMatch(/line 4.*max_out_tokens/);
    expect(err(good.replace('GSTACK_EVAL_CALLS v1: purpose=judge', 'GSTACK_EVAL_CALLS v2: purpose=judge'))).toMatch(/line 5.*version/);
    expect(err(good + 'GSTACK_EVAL_PRECONDITION: after the end\n')).toMatch(/after the end line/);
    expect(err(good.replace(/^GSTACK_EVAL_HEAD.*\n/m, ''))).toMatch(/GSTACK_EVAL_HEAD/);
  });
});

describe('price-gate: binding, pricing and the receipt (bin, simulated Capy host)', () => {
  let host: CapyHost;
  beforeEach(() => { host = capyHost(); });
  afterEach(() => host.cleanup());

  function writePlan(repo: string, plan: Parameters<typeof formatEvalCalls>[0]): string {
    const file = path.join(host.base, 'plan.calls');
    fs.writeFileSync(file, formatEvalCalls(plan));
    return file;
  }

  test('two call lines and one precondition are priced and listed, and the receipt binds price to gate, head, inputs and plan bytes', () => {
    const { repo, head, rowsSha } = scratchRepo(host.env);
    const file = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: ['ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time'], rows: twoRows });
    const out = path.join(host.base, 'receipts');
    const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '20', '--out', out]);
    expect(r.err).toBe('');
    expect(r.code).toBe(0);
    // opus-5-5: 0.4 MTok × $4 + 0.2 MTok × $20 = $5.60; sol: 0.3 × $2 + 0.05 × $10 = $1.10
    expect(r.out).toContain('GATE_CALLS: purpose=answer model=anthropic/claude-opus-5-5 calls=100 usd=5.6 reservation_usd=0.17984 scope=full');
    expect(r.out).toContain('GATE_CALLS: purpose=judge model=gpt-6.1-sol calls=100 usd=1.1 reservation_usd=0.02648 scope=full');
    expect(r.out).toContain('GATE_PRECONDITION: ANTHROPIC_API_KEY and OPENAI_API_KEY set at run time');
    const price = r.out.split('\n').find(l => l.startsWith('GATE_PRICE: '))!;
    expect(price).toMatch(/^GATE_PRICE: total=6\.7 calls=200 purposes=answer,judge receipt=\S+$/);
    const receiptPath = price.split('receipt=')[1]!;
    expect(path.dirname(receiptPath)).toBe(out);
    const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8'));
    expect(receipt).toMatchObject({
      schema_version: 1, result: 'priced', code: null, gate_command: GATE, gate_sha256: createHash('sha256').update(GATE).digest('hex'),
      head_sha: head, plan: { file, source: 'flag', declared_head: head, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex') },
      inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], total_usd: 6.7, calls: 200, cap_usd: 20, unpriced_override: null,
    });
    expect(typeof receipt.gstack_revision).toBe('string');
  });

  test('--gate-file prices the gate command the agent wrote to a file, byte for byte as --gate does; empty, missing or both refuse', () => {
    const { repo, head, rowsSha } = scratchRepo(host.env);
    const quoted = `${GATE} --label "it's \\"quoted\\" \\$HOME"`;
    const file = writePlan(repo, { gate: quoted, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
    const gateFile = path.join(host.base, 'gate-command.txt');
    fs.writeFileSync(gateFile, quoted + '\n');
    const viaFile = runGate(host, repo, ['--plan-file', file, '--gate-file', gateFile, '--out', path.join(host.base, 'a')]);
    expect(viaFile.err).toBe('');
    expect(viaFile.code).toBe(0);
    const viaFlag = runGate(host, repo, ['--plan-file', file, '--gate', quoted, '--out', path.join(host.base, 'b')]);
    const total = (out: string) => out.split('\n').find(l => l.startsWith('GATE_PRICE: '))!.replace(/ receipt=.*/, '');
    expect(total(viaFile.out)).toBe(total(viaFlag.out));
    const receipt = JSON.parse(fs.readFileSync(viaFile.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
    expect(receipt.gate_command).toBe(quoted);
    fs.writeFileSync(gateFile, '\n');
    expect(runGate(host, repo, ['--plan-file', file, '--gate-file', gateFile]).code).toBe(2);
    expect(runGate(host, repo, ['--plan-file', file, '--gate-file', path.join(host.base, 'absent')]).code).toBe(2);
    expect(runGate(host, repo, ['--plan-file', file, '--gate', quoted, '--gate-file', gateFile]).code).toBe(2);
  });

  test('the base policy price_file is read from the HEAD commit; a plan committed on top of its own head stays fresh until another file changes', () => {
    const { repo, rowsSha } = scratchRepo(host.env);
    const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 }).stdout.trim();
    fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
    fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/gate.calls' }));
    git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
    const produced = git('rev-parse', 'HEAD');
    git('update-ref', 'refs/remotes/origin/main', produced);
    fs.mkdirSync(path.join(repo, 'evals'));
    fs.writeFileSync(path.join(repo, 'evals', 'gate.calls'), formatEvalCalls({ gate: GATE, head: produced, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows }));
    git('add', 'evals/gate.calls'); git('commit', '-q', '-m', 'call plan');
    const out = path.join(host.base, 'receipts');
    let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
    expect(r.err).toBe('');
    expect(r.code).toBe(0);
    expect(r.out).toContain(`GATE_PLAN: file=evals/gate.calls source=price_file`);
    expect(r.out).toContain('only the plan file changed since');
    const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
    expect(receipt).toMatchObject({ plan: { source: 'price_file', declared_head: produced }, head_sha: git('rev-parse', 'HEAD'), total_usd: 6.7 });
    fs.writeFileSync(path.join(repo, 'README.md'), 'later\n');
    git('add', 'README.md'); git('commit', '-q', '-m', 'later');
    r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/behind HEAD.*README\.md.*\(EVAL_PLAN_STALE\)/);
  });

  test('a declared price_file missing at HEAD refuses EVAL_PLAN_STALE (not overridable); an unresolvable base or an invalid pregate.json exits 3', () => {
    const { repo } = scratchRepo(host.env);
    const git = (...args: string[]) => spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 }).stdout.trim();
    const out = path.join(host.base, 'receipts');
    let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
    expect(r.code).toBe(3);
    expect(r.err).toContain('(POLICY_SOURCE_UNAVAILABLE)');
    fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
    fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: 'evals/missing.calls' }));
    git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'policy');
    git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
    r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/evals\/missing\.calls.*\(EVAL_PLAN_STALE\)/);
    expect(r.out).not.toContain('GATE_UNPRICED_OVERRIDE');
    fs.writeFileSync(path.join(repo, '.gstack', 'pregate.json'), JSON.stringify({ price_file: '../outside.calls' }));
    git('add', '.gstack/pregate.json'); git('commit', '-q', '-m', 'bad policy');
    git('update-ref', 'refs/remotes/origin/main', git('rev-parse', 'HEAD'));
    r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out, '--allow-unpriced', 'owner said so']);
    expect(r.code).toBe(3);
    expect(r.err).toContain('(PREGATE_CONFIG_INVALID)');
    expect(fs.existsSync(out)).toBe(false);
  });

  test('an unknown or misspelled flag exits 2 instead of pricing without it', () => {
    const { repo } = scratchRepo(host.env);
    expect(runGate(host, repo, ['--gate', GATE, '--cap', '5']).code).toBe(2);
    expect(runGate(host, repo, ['--cap-usd', '5']).code).toBe(2);
  });

  test('nothing in the plan file is executed: a gate command and precondition that would create markers leave none', () => {
    const { repo, head, rowsSha } = scratchRepo(host.env);
    const marker = path.join(host.base, 'gate-ran');
    const gate = `touch ${marker}`;
    const file = writePlan(repo, { gate, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [`$(touch ${marker}-pre)`], rows: twoRows });
    const r = runGate(host, repo, ['--plan-file', file, '--gate', gate]);
    expect(r.code).toBe(0);
    expect(fs.existsSync(marker)).toBe(false);
    expect(fs.existsSync(`${marker}-pre`)).toBe(false);
  });

  test('a stale head, an input-hash mismatch and a gate-command mismatch each refuse EVAL_PLAN_STALE with no receipt', () => {
    const { repo, head, rowsSha } = scratchRepo(host.env);
    const out = path.join(host.base, 'receipts');
    const staleHead = writePlan(repo, { gate: GATE, head: 'f'.repeat(40), inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
    let r = runGate(host, repo, ['--plan-file', staleHead, '--gate', GATE, '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/head .*ffffffffffff.*\(EVAL_PLAN_STALE\)/);
    const badInput = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: '0'.repeat(64) }], preconditions: [], rows: twoRows });
    r = runGate(host, repo, ['--plan-file', badInput, '--gate', GATE, '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/rows\.jsonl.*sha256.*\(EVAL_PLAN_STALE\)/);
    const good = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: twoRows });
    r = runGate(host, repo, ['--plan-file', good, '--gate', `${GATE} --questions 500`, '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/gate command.*\(EVAL_PLAN_STALE\)/);
    expect(fs.existsSync(out)).toBe(false);
  });

  test('malformed, partial (no end line) and empty-of-binding files refuse EVAL_DRY_RUN_FAILED naming the line', () => {
    const { repo, head } = scratchRepo(host.env);
    const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
    fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace(/GSTACK_EVAL_CALLS v1 end: lines=2\n$/, ''));
    let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/end line.*\(EVAL_DRY_RUN_FAILED\)/);
    fs.writeFileSync(file, formatEvalCalls({ gate: GATE, head, inputs: [], preconditions: [], rows: twoRows }).replace('calls=100', 'calls=lots'));
    r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/line \d+.*calls.*\(EVAL_DRY_RUN_FAILED\)/);
  });

  test('no plan file, or a file with no call lines, is EVAL_GATE_UNPRICED exit 1; --allow-unpriced exits 0 with a recorded override', () => {
    const { repo, head, rowsSha } = scratchRepo(host.env);
    spawnSync('git', ['update-ref', 'refs/remotes/origin/main', head], { cwd: repo, env: host.env, timeout: 30_000 });
    const out = path.join(host.base, 'receipts');
    let r = runGate(host, repo, ['--gate', GATE, '--base', 'main', '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
    const empty = writePlan(repo, { gate: GATE, head, inputs: [{ path: 'rows.jsonl', sha256: rowsSha }], preconditions: [], rows: [] });
    r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out]);
    expect(r.code).toBe(1);
    expect(r.err).toContain('(EVAL_GATE_UNPRICED)');
    expect(r.out).toContain('GATE_PRICE: total=unknown calls=0 purposes=none');
    r = runGate(host, repo, ['--plan-file', empty, '--gate', GATE, '--out', out, '--allow-unpriced', 'owner accepts a bounded smoke on a seat']);
    expect(r.code).toBe(0);
    expect(r.out).toContain('GATE_UNPRICED_OVERRIDE: owner accepts a bounded smoke on a seat (owner-authorized override; not authorization to spend)');
    const receipt = JSON.parse(fs.readFileSync(r.out.split('receipt=')[1]!.split('\n')[0]!, 'utf8'));
    expect(receipt).toMatchObject({ result: 'unpriced_override', code: 'EVAL_GATE_UNPRICED', unpriced_override: 'owner accepts a bounded smoke on a seat', total_usd: null });
  });

  test('an unpriced model makes its row and the total unknown, never zero, and fails EVAL_PRICE_MISSING', () => {
    const { repo, head } = scratchRepo(host.env);
    const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [twoRows[0]!, { ...twoRows[1]!, model: 'acme/made-up-model' }] });
    const r = runGate(host, repo, ['--plan-file', file, '--gate', GATE]);
    expect(r.code).toBe(1);
    expect(r.out).toContain('GATE_CALLS: purpose=judge model=acme/made-up-model calls=100 usd=unknown reservation_usd=unknown scope=full');
    expect(r.out).toMatch(/GATE_PRICE: total=unknown calls=200 /);
    expect(r.err).toContain('(EVAL_PRICE_MISSING)');
  });

  test('a total above --cap-usd is EVAL_GATE_OVER_CAP; a single-call reservation above the cap or the smoke cap fails reservation-fits-cap', () => {
    const { repo, head } = scratchRepo(host.env);
    const file = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: twoRows });
    let r = runGate(host, repo, ['--plan-file', file, '--gate', GATE, '--cap-usd', '5']);
    expect(r.code).toBe(1);
    expect(r.err).toMatch(/6\.7.*cap \$5.*\(EVAL_GATE_OVER_CAP\)/);
    const smoke = writePlan(repo, { gate: GATE, head, inputs: [], preconditions: [], rows: [{ purpose: 'smoke-answer', model: 'claude-opus-5-5', calls: 2, in_tokens: 20_000, out_tokens: 4_000, max_out_tokens: 64_000, scope: 'smoke' }] });
    // one call: 10,000 in × $4 + 64,000 out × $20 per MTok = $1.32, under the $10 cap but over the $1 smoke cap
    r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '10', '--smoke-cap-usd', '1']);
    expect(r.code).toBe(1);
    expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > smoke cap $1.00) purpose=smoke-answer');
    expect(r.err).toContain('(EVAL_GATE_OVER_CAP)');
    r = runGate(host, repo, ['--plan-file', smoke, '--gate', GATE, '--cap-usd', '1.2']);
    expect(r.code).toBe(1);
    expect(r.out).toContain('CHECK: reservation-fits-cap exceeds ($1.3200 > cap $1.20) purpose=smoke-answer');
  });
});

describe('dry-run reservation-fits-cap', () => {
  test('with --cap, one call per arm (largest row input + output budget), priced, must fit under the cap', () => {
    const d = tmp();
    const rows = path.join(d, 'rows.jsonl');
    fs.writeFileSync(rows, [JSON.stringify({ prompt: 'x'.repeat(4_000) }), JSON.stringify({ prompt: 'x'.repeat(40_000) })].join('\n') + '\n');
    const retry = path.join(d, 'retry.txt'); fs.writeFileSync(retry, 'retry\n');
    // largest row 10,000 tokens; one gpt-6-astra call = 10,000 × $10 + 8,192 × $50 per MTok = $0.5096
    const tight = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 0.5 });
    expect(tight.checks.reservation_fits_cap).toMatchObject({ ok: false, cap: 0.5, max_input_tokens: 10_000 });
    expect(tight.checks.reservation_fits_cap!.per_arm['gpt-6-astra']).toBeCloseTo(0.5096, 6);
    expect(tight.pass).toBe(false);
    expect(renderDryRun(tight)).toContain('CHECK: reservation-fits-cap exceeds ($0.5096 > cap $0.50)');
    const roomy = dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry, cap: 1 });
    expect(roomy.checks.reservation_fits_cap!.ok).toBe(true);
    expect(roomy.pass).toBe(true);
    expect(dryRun({ input: rows, arms: ['gpt-6-astra'], retryPrompt: retry }).checks.reservation_fits_cap).toBeUndefined();
  });
});

describe('pricing rows with fetched sources (2026-10-12)', () => {
  test('the newest Sonnet, the newest Gemini Flash and gbrain embedding models are priced; Opus 4.7 is at its current list price', () => {
    expect(hasPricing('anthropic/claude-sonnet-5-5')).toBe(true);
    expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-sonnet-5-5')).toBe(12);
    expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'claude-opus-4-7')).toBe(30);
    expect(hasPricing('google/gemini-3.8-flash')).toBe(true);
    for (const m of ['voyage:voyage-4', 'openai:text-embedding-3-large', 'text-embedding-3-small', 'gemini-embedding-2']) expect(hasPricing(m)).toBe(true);
    expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'voyage:voyage-4')).toBe(0.06);
    expect(hasPricing('voyage-4-nano')).toBe(false);
  });

  test('every model in a gbrain `eval longmemeval --call-plan` file is priced: dated Claude snapshots, the judge and the reranker', () => {
    expect(pricingRow('anthropic:claude-haiku-4-5-20251001')).toBe(PRICING['claude-haiku-4-5']);
    expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000 }, 'openai:gpt-4o')).toBe(12.5);
    expect(hasPricing('gpt-4o-2024-05-13')).toBe(false);
    expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'voyage:rerank-2.5')).toBe(0.05);
    expect(estimateCostUsd({ input: 1_000_000, output: 0 }, 'rerank-2.5-lite')).toBe(0.02);
    expect(hasPricing('claude-haiku-4-5-2025100')).toBe(false);
  });
});