fix(benchmark): price Codex cached input at the cache rate, stop calling Gemini OAuth ready

Codex reports cached_input_tokens as a subset of input_tokens, but the
adapter billed all of input_tokens at the full input rate. Split them
into disjoint uncached and cached counts (what estimateCostUsd expects)
and show new+cached input in the table and Markdown reports. A stored
Gemini OAuth file alone is now reported as not ready, with a reason
naming GEMINI_API_KEY; key-based auth stays ready. Adopts the focused
subset of #2283; the capacity error code and stdin change are left out
(no captured failure or reproduced hang).

Co-authored-by: Mattias Petersson <mattias.petersson@gmail.com>
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
committed 2026-10-08 06:20:07 +00:00
1 parent 6a46e8c22a
commit 04359a12de
4 files changed
+105 -26

No files matched your search

+70
View File
@@ -0,0 +1,70 @@
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { spawnSync } from 'child_process';
import { resultFromCodexStream } from './helpers/providers/gpt';
import { GeminiAdapter } from './helpers/providers/gemini';
import { estimateCostUsd } from './helpers/pricing';
import { formatMarkdown, formatTable, type BenchmarkReport } from './helpers/benchmark-runner';
describe('Codex token accounting', () => {
const turn = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } });
test('cached input is split out of input_tokens and priced at the cache-read rate', () => {
const result = resultFromCodexStream(turn, { model: 'gpt-5.4' });
expect(result.tokens).toEqual({ input: 11_171, cached: 9_984, output: 5 });
// gpt-5.4: $2.50/MTok uncached, 10% of that for cache reads, $10/MTok output.
expect(estimateCostUsd(result.tokens, 'gpt-5.4')).toBeCloseTo(0.030474, 5);
});
test('turns accumulate both buckets', () => {
const second = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } });
expect(resultFromCodexStream(`${turn}\n${second}`).tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 });
});
test('reports show new and cached input', () => {
const report: BenchmarkReport = {
prompt: 'p', workdir: '/tmp', startedAt: '2026-10-07T00:00:00Z', durationMs: 1,
entries: [{ provider: 'gpt', family: 'gpt', available: true, result: resultFromCodexStream(turn, { model: 'gpt-5.4' }), costUsd: 0.03 }],
};
expect(formatTable(report)).toContain('11171+9984→5');
expect(formatMarkdown(report)).toContain('11171+9984→5');
});
});
describe('Gemini availability', () => {
let home: string;
beforeEach(() => {
home = fs.mkdtempSync(path.join(os.tmpdir(), 'gemini-avail-'));
fs.mkdirSync(path.join(home, 'bin'));
fs.writeFileSync(path.join(home, 'bin', 'gemini'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
fs.mkdirSync(path.join(home, '.gemini'));
fs.writeFileSync(path.join(home, '.gemini', 'oauth_creds.json'), '{}');
});
afterEach(() => fs.rmSync(home, { recursive: true, force: true }));
// Bun fixes os.homedir() at startup, so each check runs in a child with the temp HOME.
function availability(extraEnv: Record<string, string> = {}): { ok: boolean; reason?: string } {
const env: Record<string, string> = { ...(process.env as Record<string, string>), HOME: home, PATH: `${path.join(home, 'bin')}:${process.env.PATH}`, ...extraEnv };
delete env.GOOGLE_API_KEY;
if (!extraEnv.GEMINI_API_KEY) delete env.GEMINI_API_KEY;
const script = `const { GeminiAdapter } = await import(${JSON.stringify(path.join(import.meta.dir, 'helpers/providers/gemini.ts'))}); console.log(JSON.stringify(await new GeminiAdapter().available()));`;
const child = spawnSync(process.execPath, ['-e', script], { env, encoding: 'utf-8', timeout: 30_000 });
expect(child.status).toBe(0);
return JSON.parse(child.stdout.trim().split('\n').pop()!);
}
test('stored OAuth alone is not ready and names the API key fix', () => {
const check = availability();
expect(check.ok).toBe(false);
expect(check.reason).toContain('stored Gemini OAuth');
expect(check.reason).toContain('GEMINI_API_KEY');
});
test('an API key is ready', () => {
expect(availability({ GEMINI_API_KEY: 'fixture-not-a-key' })).toEqual({ ok: true });
});
});
+15 -9
View File
@@ -7,7 +7,7 @@
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
*/
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types';
import { ClaudeAdapter } from './providers/claude';
import { GptAdapter } from './providers/gpt';
import { GeminiAdapter } from './providers/gemini';
@@ -99,21 +99,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkRepo
}
export function formatTable(report: BenchmarkReport): string {
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
const header = `Model Latency In(+cached)→Out Tokens Cost Quality Tool Calls Notes`;
const sep = '-'.repeat(header.length);
const rows: string[] = [header, sep];
for (const e of report.entries) {
if (!e.available) {
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 24)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
continue;
}
const r = e.result!;
if (r.error) {
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
continue;
}
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
}
return rows.join('\n');
}
@@ -130,8 +130,8 @@ export function formatMarkdown(report: BenchmarkReport): string {
`**Workdir:** \`${report.workdir}\``,
`**Total duration:** ${msToStr(report.durationMs)}`,
'',
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
'|-------|---------|-----------------|------|---------|-------|-------|',
'| Model | Latency | Tokens (in+cached→out) | Cost | Quality | Tools | Notes |',
'|-------|---------|------------------------|------|---------|-------|-------|',
];
for (const e of report.entries) {
if (!e.available) {
@@ -140,11 +140,11 @@ export function formatMarkdown(report: BenchmarkReport): string {
}
const r = e.result!;
if (r.error) {
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
continue;
}
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
}
return lines.join('\n');
}
@@ -153,6 +153,12 @@ function pad(s: string, n: number): string {
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
}
function fmtTokens(tokens: TokenUsage): string {
return tokens.cached === undefined
? `${tokens.input}→${tokens.output}`
: `${tokens.input}+${tokens.cached}→${tokens.output}`;
}
function msToStr(ms: number): string {
if (ms < 1000) return `${ms}ms`;
return `${(ms / 1000).toFixed(1)}s`;
+12 -13
View File
@@ -109,9 +109,10 @@ export function resultFromGeminiStream(
/**
* Gemini adapter — wraps the `gemini` CLI.
*
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
* Personal OAuth free-tier is no longer supported by gemini CLI — use an
* AI Studio API key. Antigravity is a separate product/quota path.
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY. Stored ~/.gemini OAuth alone is
* reported as not ready: personal OAuth free-tier is no longer supported by
* gemini CLI — use an AI Studio API key. Antigravity is a separate
* product/quota path.
*
* Headless flags always passed:
* --output-format stream-json — NDJSON events (message/tool_use/result)
@@ -134,17 +135,15 @@ export class GeminiAdapter implements ProviderAdapter {
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
const newCfgDir = path.join(os.homedir(), '.gemini');
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
if (!hasCfg && !hasKey) {
return {
ok: false,
reason:
'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
};
}
return { ok: true };
if (process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY) return { ok: true };
const hasOauth = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
return {
ok: false,
reason: hasOauth
? 'Only stored Gemini OAuth found; personal OAuth free-tier is no longer supported by gemini CLI and headless runs can fail. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey.'
: 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
};
}
async run(opts: RunOpts): Promise<RunResult> {
+8 -4
View File
@@ -99,9 +99,13 @@ export class GptAdapter implements ProviderAdapter {
}
}
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
/**
* Map `codex exec --json` output while requiring evidence-bearing text for paid producers.
* Codex reports cached_input_tokens as a subset of input_tokens; split them so
* tokens.input is uncached-only and tokens.cached is the disjoint cache-read count.
*/
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
let output='',input=0,cached=0,out=0,toolCalls=0,modelUsed:string|undefined;
for(const line of raw.split('\n')){
const text=line.trim();if(!text)continue;
try{
@@ -110,14 +114,14 @@ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
else if(obj.item.type==='command_execution')toolCalls++;
}else if(obj.type==='turn.completed'){
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
const usage=obj.usage??{},turnCached=usage.cached_input_tokens??0;input+=Math.max(0,(usage.input_tokens??0)-turnCached);cached+=turnCached;out+=usage.output_tokens??0;
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
}
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
}
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
return{output,tokens:{input,cached,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
}
export interface CodexProducerPaths {