diff --git a/test/benchmark-provider-adapters.test.ts b/test/benchmark-provider-adapters.test.ts new file mode 100644 index 000000000..af3cd11d3 --- /dev/null +++ b/test/benchmark-provider-adapters.test.ts @@ -0,0 +1,70 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { resultFromCodexStream } from './helpers/providers/gpt'; +import { GeminiAdapter } from './helpers/providers/gemini'; +import { estimateCostUsd } from './helpers/pricing'; +import { formatMarkdown, formatTable, type BenchmarkReport } from './helpers/benchmark-runner'; + +describe('Codex token accounting', () => { + const turn = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } }); + + test('cached input is split out of input_tokens and priced at the cache-read rate', () => { + const result = resultFromCodexStream(turn, { model: 'gpt-5.4' }); + expect(result.tokens).toEqual({ input: 11_171, cached: 9_984, output: 5 }); + // gpt-5.4: $2.50/MTok uncached, 10% of that for cache reads, $10/MTok output. + expect(estimateCostUsd(result.tokens, 'gpt-5.4')).toBeCloseTo(0.030474, 5); + }); + + test('turns accumulate both buckets', () => { + const second = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } }); + expect(resultFromCodexStream(`${turn}\n${second}`).tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 }); + }); + + test('reports show new and cached input', () => { + const report: BenchmarkReport = { + prompt: 'p', workdir: '/tmp', startedAt: '2026-10-07T00:00:00Z', durationMs: 1, + entries: [{ provider: 'gpt', family: 'gpt', available: true, result: resultFromCodexStream(turn, { model: 'gpt-5.4' }), costUsd: 0.03 }], + }; + expect(formatTable(report)).toContain('11171+9984→5'); + expect(formatMarkdown(report)).toContain('11171+9984→5'); + }); +}); + +describe('Gemini availability', () => { + let home: string; + + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'gemini-avail-')); + fs.mkdirSync(path.join(home, 'bin')); + fs.writeFileSync(path.join(home, 'bin', 'gemini'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + fs.mkdirSync(path.join(home, '.gemini')); + fs.writeFileSync(path.join(home, '.gemini', 'oauth_creds.json'), '{}'); + }); + + afterEach(() => fs.rmSync(home, { recursive: true, force: true })); + + // Bun fixes os.homedir() at startup, so each check runs in a child with the temp HOME. + function availability(extraEnv: Record = {}): { ok: boolean; reason?: string } { + const env: Record = { ...(process.env as Record), HOME: home, PATH: `${path.join(home, 'bin')}:${process.env.PATH}`, ...extraEnv }; + delete env.GOOGLE_API_KEY; + if (!extraEnv.GEMINI_API_KEY) delete env.GEMINI_API_KEY; + const script = `const { GeminiAdapter } = await import(${JSON.stringify(path.join(import.meta.dir, 'helpers/providers/gemini.ts'))}); console.log(JSON.stringify(await new GeminiAdapter().available()));`; + const child = spawnSync(process.execPath, ['-e', script], { env, encoding: 'utf-8', timeout: 30_000 }); + expect(child.status).toBe(0); + return JSON.parse(child.stdout.trim().split('\n').pop()!); + } + + test('stored OAuth alone is not ready and names the API key fix', () => { + const check = availability(); + expect(check.ok).toBe(false); + expect(check.reason).toContain('stored Gemini OAuth'); + expect(check.reason).toContain('GEMINI_API_KEY'); + }); + + test('an API key is ready', () => { + expect(availability({ GEMINI_API_KEY: 'fixture-not-a-key' })).toEqual({ ok: true }); + }); +}); diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index cbef4107b..324cce432 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -7,7 +7,7 @@ * one. Per-provider auth/timeout/rate-limit errors don't abort the batch. */ -import type { ProviderAdapter, RunOpts, RunResult } from './providers/types'; +import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types'; import { ClaudeAdapter } from './providers/claude'; import { GptAdapter } from './providers/gpt'; import { GeminiAdapter } from './providers/gemini'; @@ -99,21 +99,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise= n ? s.slice(0, n) : s + ' '.repeat(n - s.length); } +function fmtTokens(tokens: TokenUsage): string { + return tokens.cached === undefined + ? `${tokens.input}→${tokens.output}` + : `${tokens.input}+${tokens.cached}→${tokens.output}`; +} + function msToStr(ms: number): string { if (ms < 1000) return `${ms}ms`; return `${(ms / 1000).toFixed(1)}s`; diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 4833e6d12..8882f92af 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -109,9 +109,10 @@ export function resultFromGeminiStream( /** * Gemini adapter — wraps the `gemini` CLI. * - * Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth. - * Personal OAuth free-tier is no longer supported by gemini CLI — use an - * AI Studio API key. Antigravity is a separate product/quota path. + * Auth: GEMINI_API_KEY / GOOGLE_API_KEY. Stored ~/.gemini OAuth alone is + * reported as not ready: personal OAuth free-tier is no longer supported by + * gemini CLI — use an AI Studio API key. Antigravity is a separate + * product/quota path. * * Headless flags always passed: * --output-format stream-json — NDJSON events (message/tool_use/result) @@ -134,17 +135,15 @@ export class GeminiAdapter implements ProviderAdapter { const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini'); const newCfgDir = path.join(os.homedir(), '.gemini'); const newOauth = path.join(newCfgDir, 'oauth_creds.json'); - const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth)); // CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY. - const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY); - if (!hasCfg && !hasKey) { - return { - ok: false, - reason: - 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.', - }; - } - return { ok: true }; + if (process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY) return { ok: true }; + const hasOauth = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth)); + return { + ok: false, + reason: hasOauth + ? 'Only stored Gemini OAuth found; personal OAuth free-tier is no longer supported by gemini CLI and headless runs can fail. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey.' + : 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.', + }; } async run(opts: RunOpts): Promise { diff --git a/test/helpers/providers/gpt.ts b/test/helpers/providers/gpt.ts index 200a431e1..24c92f991 100644 --- a/test/helpers/providers/gpt.ts +++ b/test/helpers/providers/gpt.ts @@ -99,9 +99,13 @@ export class GptAdapter implements ProviderAdapter { } } -/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */ +/** + * Map `codex exec --json` output while requiring evidence-bearing text for paid producers. + * Codex reports cached_input_tokens as a subset of input_tokens; split them so + * tokens.input is uncached-only and tokens.cached is the disjoint cache-read count. + */ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{ - let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined; + let output='',input=0,cached=0,out=0,toolCalls=0,modelUsed:string|undefined; for(const line of raw.split('\n')){ const text=line.trim();if(!text)continue; try{ @@ -110,14 +114,14 @@ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs? if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text; else if(obj.item.type==='command_execution')toolCalls++; }else if(obj.type==='turn.completed'){ - const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0; + const usage=obj.usage??{},turnCached=usage.cached_input_tokens??0;input+=Math.max(0,(usage.input_tokens??0)-turnCached);cached+=turnCached;out+=usage.output_tokens??0; if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model; } }catch{/* Codex can mix diagnostic text into the JSONL stream. */} } const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL; if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}}; - return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel}; + return{output,tokens:{input,cached,output:out},durationMs,toolCalls,modelUsed:resolvedModel}; } export interface CodexProducerPaths {