mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 15:13:47 +01:00
fix(benchmark): price Codex cached input at the cache rate, stop calling Gemini OAuth ready
Codex reports cached_input_tokens as a subset of input_tokens, but the adapter billed all of input_tokens at the full input rate. Split them into disjoint uncached and cached counts (what estimateCostUsd expects) and show new+cached input in the table and Markdown reports. A stored Gemini OAuth file alone is now reported as not ready, with a reason naming GEMINI_API_KEY; key-based auth stays ready. Adopts the focused subset of #2283; the capacity error code and stdin change are left out (no captured failure or reproduced hang). Co-authored-by: Mattias Petersson <mattias.petersson@gmail.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
6a46e8c22a
commit
04359a12de
4 files changed
+105
-26
No files matched your search
@@ -7,7 +7,7 @@
|
||||
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
|
||||
*/
|
||||
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
@@ -99,21 +99,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkRepo
|
||||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const header = `Model Latency In(+cached)→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 24)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
@@ -130,8 +130,8 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
||||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
'| Model | Latency | Tokens (in+cached→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|------------------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
@@ -140,11 +140,11 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -153,6 +153,12 @@ function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function fmtTokens(tokens: TokenUsage): string {
|
||||
return tokens.cached === undefined
|
||||
? `${tokens.input}→${tokens.output}`
|
||||
: `${tokens.input}+${tokens.cached}→${tokens.output}`;
|
||||
}
|
||||
|
||||
function msToStr(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
|
||||
@@ -109,9 +109,10 @@ export function resultFromGeminiStream(
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
*
|
||||
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
|
||||
* Personal OAuth free-tier is no longer supported by gemini CLI — use an
|
||||
* AI Studio API key. Antigravity is a separate product/quota path.
|
||||
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY. Stored ~/.gemini OAuth alone is
|
||||
* reported as not ready: personal OAuth free-tier is no longer supported by
|
||||
* gemini CLI — use an AI Studio API key. Antigravity is a separate
|
||||
* product/quota path.
|
||||
*
|
||||
* Headless flags always passed:
|
||||
* --output-format stream-json — NDJSON events (message/tool_use/result)
|
||||
@@ -134,17 +135,15 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
|
||||
const newCfgDir = path.join(os.homedir(), '.gemini');
|
||||
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
|
||||
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
|
||||
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
|
||||
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
|
||||
if (!hasCfg && !hasKey) {
|
||||
return {
|
||||
ok: false,
|
||||
reason:
|
||||
'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
if (process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY) return { ok: true };
|
||||
const hasOauth = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
|
||||
return {
|
||||
ok: false,
|
||||
reason: hasOauth
|
||||
? 'Only stored Gemini OAuth found; personal OAuth free-tier is no longer supported by gemini CLI and headless runs can fail. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey.'
|
||||
: 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
|
||||
};
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
|
||||
@@ -99,9 +99,13 @@ export class GptAdapter implements ProviderAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
|
||||
/**
|
||||
* Map `codex exec --json` output while requiring evidence-bearing text for paid producers.
|
||||
* Codex reports cached_input_tokens as a subset of input_tokens; split them so
|
||||
* tokens.input is uncached-only and tokens.cached is the disjoint cache-read count.
|
||||
*/
|
||||
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
|
||||
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
|
||||
let output='',input=0,cached=0,out=0,toolCalls=0,modelUsed:string|undefined;
|
||||
for(const line of raw.split('\n')){
|
||||
const text=line.trim();if(!text)continue;
|
||||
try{
|
||||
@@ -110,14 +114,14 @@ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?
|
||||
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
|
||||
else if(obj.item.type==='command_execution')toolCalls++;
|
||||
}else if(obj.type==='turn.completed'){
|
||||
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
|
||||
const usage=obj.usage??{},turnCached=usage.cached_input_tokens??0;input+=Math.max(0,(usage.input_tokens??0)-turnCached);cached+=turnCached;out+=usage.output_tokens??0;
|
||||
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
|
||||
}
|
||||
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
|
||||
}
|
||||
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
|
||||
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
|
||||
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
|
||||
return{output,tokens:{input,cached,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
|
||||
}
|
||||
|
||||
export interface CodexProducerPaths {
|
||||
|
||||
Reference in new issue
Block a user