mirror of
https://github.com/garrytan/gstack.git
synced 2026-07-18 21:47:20 +02:00
fix(benchmark): parse current gemini stream-json content/stats shape
GeminiAdapter was reading message.text and result.usage, so current CLI content/stats events produced empty $0 success rows. Accept content with an assistant role guard, stats token fallbacks, init model, and treat empty exit-0 output as an error (#2159). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Sina
@@ -0,0 +1,95 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { parseGeminiStreamJson } from './gemini';
|
||||
|
||||
// Current CLI shape (gemini ≥ ~1.58): content + role, tokens under result.stats
|
||||
const CURRENT_CLI_FIXTURE = [
|
||||
'{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}',
|
||||
'{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}',
|
||||
'{"type":"message","role":"assistant","content":"PONG","delta":true}',
|
||||
'{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}',
|
||||
].join('\n');
|
||||
|
||||
// Legacy shape: text field, tokens under result.usage
|
||||
const LEGACY_FIXTURE = [
|
||||
'{"type":"message","text":"hello"}',
|
||||
'{"type":"tool_use","name":"run_shell_command"}',
|
||||
'{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}',
|
||||
].join('\n');
|
||||
|
||||
describe('parseGeminiStreamJson', () => {
|
||||
test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => {
|
||||
const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE);
|
||||
expect(parsed.output).toBe('PONG');
|
||||
expect(parsed.tokens.input).toBe(24946);
|
||||
expect(parsed.tokens.output).toBe(32);
|
||||
expect(parsed.modelUsed).toBe('gemini-2.5-pro');
|
||||
expect(parsed.toolCalls).toBe(0);
|
||||
});
|
||||
|
||||
test('current CLI: user role content must not concat into output', () => {
|
||||
const raw = [
|
||||
'{"type":"message","role":"user","content":"PROMPT ECHO"}',
|
||||
'{"type":"message","role":"assistant","content":"ok","delta":true}',
|
||||
].join('\n');
|
||||
const parsed = parseGeminiStreamJson(raw);
|
||||
expect(parsed.output).toBe('ok');
|
||||
expect(parsed.output).not.toContain('PROMPT ECHO');
|
||||
});
|
||||
|
||||
test('legacy: still accepts text + usage.input_token_count', () => {
|
||||
const parsed = parseGeminiStreamJson(LEGACY_FIXTURE);
|
||||
expect(parsed.output).toBe('hello');
|
||||
expect(parsed.tokens.input).toBe(100);
|
||||
expect(parsed.tokens.output).toBe(5);
|
||||
expect(parsed.toolCalls).toBe(1);
|
||||
expect(parsed.modelUsed).toBe('gemini-2.5-flash');
|
||||
});
|
||||
|
||||
test('legacy text with role:user is ignored', () => {
|
||||
const raw = [
|
||||
'{"type":"message","role":"user","text":"echo"}',
|
||||
'{"type":"message","role":"assistant","text":"kept"}',
|
||||
].join('\n');
|
||||
const parsed = parseGeminiStreamJson(raw);
|
||||
expect(parsed.output).toBe('kept');
|
||||
});
|
||||
|
||||
test('concatenates multiple assistant content deltas', () => {
|
||||
const raw = [
|
||||
'{"type":"message","role":"assistant","content":"A","delta":true}',
|
||||
'{"type":"message","role":"assistant","content":"B","delta":true}',
|
||||
].join('\n');
|
||||
expect(parseGeminiStreamJson(raw).output).toBe('AB');
|
||||
});
|
||||
|
||||
test('skips malformed lines without throwing', () => {
|
||||
const raw = [
|
||||
'{"type":"init","model":"m1"}',
|
||||
'not json',
|
||||
'{"type":"message","role":"assistant","content":"x","delta":true}',
|
||||
'{incomplete',
|
||||
'{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}',
|
||||
].join('\n');
|
||||
const parsed = parseGeminiStreamJson(raw);
|
||||
expect(parsed.output).toBe('x');
|
||||
expect(parsed.tokens).toEqual({ input: 1, output: 2 });
|
||||
expect(parsed.modelUsed).toBe('m1');
|
||||
});
|
||||
|
||||
test('empty / whitespace-only input yields empty parse (no throw)', () => {
|
||||
const parsed = parseGeminiStreamJson('');
|
||||
expect(parsed.output).toBe('');
|
||||
expect(parsed.tokens).toEqual({ input: 0, output: 0 });
|
||||
expect(parsed.toolCalls).toBe(0);
|
||||
expect(parsed.modelUsed).toBeUndefined();
|
||||
});
|
||||
|
||||
test('result.model overrides init.model when both present', () => {
|
||||
const raw = [
|
||||
'{"type":"init","model":"from-init"}',
|
||||
'{"type":"message","role":"assistant","content":"hi"}',
|
||||
'{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}',
|
||||
].join('\n');
|
||||
expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result');
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,63 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
export type GeminiStreamParse = {
|
||||
output: string;
|
||||
tokens: { input: number; output: number };
|
||||
toolCalls: number;
|
||||
modelUsed?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse gemini NDJSON stream events (exported for unit tests).
|
||||
*
|
||||
* Current CLI (`--output-format stream-json`) emits:
|
||||
* init → model
|
||||
* message { role, content, delta? } → concat assistant content
|
||||
* tool_use → increment toolCalls
|
||||
* result { stats: { input_tokens, output_tokens } } → tokens
|
||||
*
|
||||
* Legacy shape (still accepted):
|
||||
* message { text } → concat text
|
||||
* result { usage: { input_token_count, output_token_count } } → tokens
|
||||
*/
|
||||
export function parseGeminiStreamJson(raw: string): GeminiStreamParse {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'init') {
|
||||
if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
|
||||
} else if (obj.type === 'message') {
|
||||
// Current CLI: content + role. Role guard is required — the CLI echoes
|
||||
// the user prompt as role:'user', which must not land in output.
|
||||
if (obj.role === 'assistant' && typeof obj.content === 'string') {
|
||||
output += obj.content;
|
||||
} else if (typeof obj.text === 'string' && obj.role !== 'user') {
|
||||
// Legacy text field (no role, or assistant).
|
||||
output += obj.text;
|
||||
}
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? obj.stats ?? {};
|
||||
input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0;
|
||||
out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0;
|
||||
if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
*
|
||||
@@ -48,13 +105,23 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
const parsed = this.parseStreamJson(out);
|
||||
const parsed = parseGeminiStreamJson(out);
|
||||
const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro';
|
||||
// Empty-success is indistinguishable from a healthy cheap run in the
|
||||
// comparison table — never report it as a $0 success row (#2159).
|
||||
if (!parsed.output.trim()) {
|
||||
return this.emptyResult(
|
||||
Date.now() - start,
|
||||
{ code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' },
|
||||
modelUsed,
|
||||
);
|
||||
}
|
||||
return {
|
||||
output: parsed.output,
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
|
||||
modelUsed,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
@@ -77,41 +144,6 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse gemini NDJSON stream events:
|
||||
* init → session id (discarded here)
|
||||
* message { delta: true, text } → concat to output
|
||||
* tool_use { name } → increment toolCalls
|
||||
* result { usage: { input_token_count, output_token_count } } → tokens
|
||||
*/
|
||||
private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
for (const line of raw.split('\n')) {
|
||||
const s = line.trim();
|
||||
if (!s) continue;
|
||||
try {
|
||||
const obj = JSON.parse(s);
|
||||
if (obj.type === 'message' && typeof obj.text === 'string') {
|
||||
output += obj.text;
|
||||
} else if (obj.type === 'tool_use') {
|
||||
toolCalls += 1;
|
||||
} else if (obj.type === 'result') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_token_count ?? u.prompt_tokens ?? 0;
|
||||
out += u.output_token_count ?? u.completion_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
return {
|
||||
output: '',
|
||||
|
||||
@@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
if (result.error) {
|
||||
throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`);
|
||||
}
|
||||
// Gemini CLI occasionally returns empty output even on successful runs
|
||||
// (model returned content the CLI parser missed, intermittent stream issues).
|
||||
// We assert the adapter ran end-to-end without erroring and reports a non-
|
||||
// empty token count instead of grepping the literal "ok" — that string
|
||||
// assertion was too brittle for a smoke that's really about "did the
|
||||
// adapter wire up and the run terminate successfully?"
|
||||
expect(typeof result.output).toBe('string');
|
||||
// Gemini CLI sometimes returns 0 tokens in the result event (older responses);
|
||||
// assert non-negative instead of strictly positive.
|
||||
expect(result.tokens.input).toBeGreaterThanOrEqual(0);
|
||||
expect(result.tokens.output).toBeGreaterThanOrEqual(0);
|
||||
// Adapter must never report empty-success (#2159). After content/stats
|
||||
// parsing, a healthy run has non-empty assistant text + token counts.
|
||||
expect(result.output.trim().length).toBeGreaterThan(0);
|
||||
expect(result.output.toLowerCase()).toContain('ok');
|
||||
expect(result.tokens.input).toBeGreaterThan(0);
|
||||
expect(result.tokens.output).toBeGreaterThan(0);
|
||||
expect(result.durationMs).toBeGreaterThan(0);
|
||||
expect(typeof result.modelUsed).toBe('string');
|
||||
expect(result.modelUsed.length).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
|
||||
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
|
||||
|
||||
Reference in New Issue
Block a user