fix(benchmark): price Codex cached input at the cache rate, stop calling Gemini OAuth ready

Codex reports cached_input_tokens as a subset of input_tokens, but the
adapter billed all of input_tokens at the full input rate. Split them
into disjoint uncached and cached counts (what estimateCostUsd expects)
and show new+cached input in the table and Markdown reports. A stored
Gemini OAuth file alone is now reported as not ready, with a reason
naming GEMINI_API_KEY; key-based auth stays ready. Adopts the focused
subset of #2283; the capacity error code and stdin change are left out
(no captured failure or reproduced hang).

Co-authored-by: Mattias Petersson <mattias.petersson@gmail.com>
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
committed 2026-10-08 06:20:07 +00:00
1 parent 6a46e8c22a
commit 04359a12de
4 files changed
+105 -26

No files matched your search

+15 -9
View File
@@ -7,7 +7,7 @@
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
*/
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types';
import { ClaudeAdapter } from './providers/claude';
import { GptAdapter } from './providers/gpt';
import { GeminiAdapter } from './providers/gemini';
@@ -99,21 +99,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkRepo
}
export function formatTable(report: BenchmarkReport): string {
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
const header = `Model Latency In(+cached)→Out Tokens Cost Quality Tool Calls Notes`;
const sep = '-'.repeat(header.length);
const rows: string[] = [header, sep];
for (const e of report.entries) {
if (!e.available) {
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 24)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
continue;
}
const r = e.result!;
if (r.error) {
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
continue;
}
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
}
return rows.join('\n');
}
@@ -130,8 +130,8 @@ export function formatMarkdown(report: BenchmarkReport): string {
`**Workdir:** \`${report.workdir}\``,
`**Total duration:** ${msToStr(report.durationMs)}`,
'',
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
'|-------|---------|-----------------|------|---------|-------|-------|',
'| Model | Latency | Tokens (in+cached→out) | Cost | Quality | Tools | Notes |',
'|-------|---------|------------------------|------|---------|-------|-------|',
];
for (const e of report.entries) {
if (!e.available) {
@@ -140,11 +140,11 @@ export function formatMarkdown(report: BenchmarkReport): string {
}
const r = e.result!;
if (r.error) {
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
continue;
}
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
}
return lines.join('\n');
}
@@ -153,6 +153,12 @@ function pad(s: string, n: number): string {
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
}
function fmtTokens(tokens: TokenUsage): string {
return tokens.cached === undefined
? `${tokens.input}→${tokens.output}`
: `${tokens.input}+${tokens.cached}→${tokens.output}`;
}
function msToStr(ms: number): string {
if (ms < 1000) return `${ms}ms`;
return `${(ms / 1000).toFixed(1)}s`;
+12 -13
View File
@@ -109,9 +109,10 @@ export function resultFromGeminiStream(
/**
* Gemini adapter — wraps the `gemini` CLI.
*
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
* Personal OAuth free-tier is no longer supported by gemini CLI — use an
* AI Studio API key. Antigravity is a separate product/quota path.
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY. Stored ~/.gemini OAuth alone is
* reported as not ready: personal OAuth free-tier is no longer supported by
* gemini CLI — use an AI Studio API key. Antigravity is a separate
* product/quota path.
*
* Headless flags always passed:
* --output-format stream-json — NDJSON events (message/tool_use/result)
@@ -134,17 +135,15 @@ export class GeminiAdapter implements ProviderAdapter {
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
const newCfgDir = path.join(os.homedir(), '.gemini');
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
if (!hasCfg && !hasKey) {
return {
ok: false,
reason:
'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
};
}
return { ok: true };
if (process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY) return { ok: true };
const hasOauth = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
return {
ok: false,
reason: hasOauth
? 'Only stored Gemini OAuth found; personal OAuth free-tier is no longer supported by gemini CLI and headless runs can fail. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey.'
: 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
};
}
async run(opts: RunOpts): Promise<RunResult> {
+8 -4
View File
@@ -99,9 +99,13 @@ export class GptAdapter implements ProviderAdapter {
}
}
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
/**
* Map `codex exec --json` output while requiring evidence-bearing text for paid producers.
* Codex reports cached_input_tokens as a subset of input_tokens; split them so
* tokens.input is uncached-only and tokens.cached is the disjoint cache-read count.
*/
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
let output='',input=0,cached=0,out=0,toolCalls=0,modelUsed:string|undefined;
for(const line of raw.split('\n')){
const text=line.trim();if(!text)continue;
try{
@@ -110,14 +114,14 @@ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
else if(obj.item.type==='command_execution')toolCalls++;
}else if(obj.type==='turn.completed'){
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
const usage=obj.usage??{},turnCached=usage.cached_input_tokens??0;input+=Math.max(0,(usage.input_tokens??0)-turnCached);cached+=turnCached;out+=usage.output_tokens??0;
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
}
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
}
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
return{output,tokens:{input,cached,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
}
export interface CodexProducerPaths {