diff --git a/test/helpers/providers/gemini.test.ts b/test/helpers/providers/gemini.test.ts new file mode 100644 index 000000000..421e00c64 --- /dev/null +++ b/test/helpers/providers/gemini.test.ts @@ -0,0 +1,95 @@ +import { describe, test, expect } from 'bun:test'; +import { parseGeminiStreamJson } from './gemini'; + +// Current CLI shape (gemini ≥ ~1.58): content + role, tokens under result.stats +const CURRENT_CLI_FIXTURE = [ + '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}', + '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}', + '{"type":"message","role":"assistant","content":"PONG","delta":true}', + '{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}', +].join('\n'); + +// Legacy shape: text field, tokens under result.usage +const LEGACY_FIXTURE = [ + '{"type":"message","text":"hello"}', + '{"type":"tool_use","name":"run_shell_command"}', + '{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}', +].join('\n'); + +describe('parseGeminiStreamJson', () => { + test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => { + const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE); + expect(parsed.output).toBe('PONG'); + expect(parsed.tokens.input).toBe(24946); + expect(parsed.tokens.output).toBe(32); + expect(parsed.modelUsed).toBe('gemini-2.5-pro'); + expect(parsed.toolCalls).toBe(0); + }); + + test('current CLI: user role content must not concat into output', () => { + const raw = [ + '{"type":"message","role":"user","content":"PROMPT ECHO"}', + '{"type":"message","role":"assistant","content":"ok","delta":true}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('ok'); + expect(parsed.output).not.toContain('PROMPT ECHO'); + }); + + test('legacy: still accepts text + usage.input_token_count', () => { + const parsed = parseGeminiStreamJson(LEGACY_FIXTURE); + expect(parsed.output).toBe('hello'); + expect(parsed.tokens.input).toBe(100); + expect(parsed.tokens.output).toBe(5); + expect(parsed.toolCalls).toBe(1); + expect(parsed.modelUsed).toBe('gemini-2.5-flash'); + }); + + test('legacy text with role:user is ignored', () => { + const raw = [ + '{"type":"message","role":"user","text":"echo"}', + '{"type":"message","role":"assistant","text":"kept"}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('kept'); + }); + + test('concatenates multiple assistant content deltas', () => { + const raw = [ + '{"type":"message","role":"assistant","content":"A","delta":true}', + '{"type":"message","role":"assistant","content":"B","delta":true}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).output).toBe('AB'); + }); + + test('skips malformed lines without throwing', () => { + const raw = [ + '{"type":"init","model":"m1"}', + 'not json', + '{"type":"message","role":"assistant","content":"x","delta":true}', + '{incomplete', + '{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('x'); + expect(parsed.tokens).toEqual({ input: 1, output: 2 }); + expect(parsed.modelUsed).toBe('m1'); + }); + + test('empty / whitespace-only input yields empty parse (no throw)', () => { + const parsed = parseGeminiStreamJson(''); + expect(parsed.output).toBe(''); + expect(parsed.tokens).toEqual({ input: 0, output: 0 }); + expect(parsed.toolCalls).toBe(0); + expect(parsed.modelUsed).toBeUndefined(); + }); + + test('result.model overrides init.model when both present', () => { + const raw = [ + '{"type":"init","model":"from-init"}', + '{"type":"message","role":"assistant","content":"hi"}', + '{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result'); + }); +}); diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 5e7abba13..cf3ad333b 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -5,6 +5,63 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +export type GeminiStreamParse = { + output: string; + tokens: { input: number; output: number }; + toolCalls: number; + modelUsed?: string; +}; + +/** + * Parse gemini NDJSON stream events (exported for unit tests). + * + * Current CLI (`--output-format stream-json`) emits: + * init → model + * message { role, content, delta? } → concat assistant content + * tool_use → increment toolCalls + * result { stats: { input_tokens, output_tokens } } → tokens + * + * Legacy shape (still accepted): + * message { text } → concat text + * result { usage: { input_token_count, output_token_count } } → tokens + */ +export function parseGeminiStreamJson(raw: string): GeminiStreamParse { + let output = ''; + let input = 0; + let out = 0; + let toolCalls = 0; + let modelUsed: string | undefined; + for (const line of raw.split('\n')) { + const s = line.trim(); + if (!s) continue; + try { + const obj = JSON.parse(s); + if (obj.type === 'init') { + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } else if (obj.type === 'message') { + // Current CLI: content + role. Role guard is required — the CLI echoes + // the user prompt as role:'user', which must not land in output. + if (obj.role === 'assistant' && typeof obj.content === 'string') { + output += obj.content; + } else if (typeof obj.text === 'string' && obj.role !== 'user') { + // Legacy text field (no role, or assistant). + output += obj.text; + } + } else if (obj.type === 'tool_use') { + toolCalls += 1; + } else if (obj.type === 'result') { + const u = obj.usage ?? obj.stats ?? {}; + input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0; + out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0; + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } + } catch { + // skip malformed lines + } + } + return { output, tokens: { input, output: out }, toolCalls, modelUsed }; +} + /** * Gemini adapter — wraps the `gemini` CLI. * @@ -48,13 +105,23 @@ export class GeminiAdapter implements ProviderAdapter { encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, }); - const parsed = this.parseStreamJson(out); + const parsed = parseGeminiStreamJson(out); + const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro'; + // Empty-success is indistinguishable from a healthy cheap run in the + // comparison table — never report it as a $0 success row (#2159). + if (!parsed.output.trim()) { + return this.emptyResult( + Date.now() - start, + { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' }, + modelUsed, + ); + } return { output: parsed.output, tokens: parsed.tokens, durationMs: Date.now() - start, toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro', + modelUsed, }; } catch (err: unknown) { const durationMs = Date.now() - start; @@ -77,41 +144,6 @@ export class GeminiAdapter implements ProviderAdapter { return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro'); } - /** - * Parse gemini NDJSON stream events: - * init → session id (discarded here) - * message { delta: true, text } → concat to output - * tool_use { name } → increment toolCalls - * result { usage: { input_token_count, output_token_count } } → tokens - */ - private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { - let output = ''; - let input = 0; - let out = 0; - let toolCalls = 0; - let modelUsed: string | undefined; - for (const line of raw.split('\n')) { - const s = line.trim(); - if (!s) continue; - try { - const obj = JSON.parse(s); - if (obj.type === 'message' && typeof obj.text === 'string') { - output += obj.text; - } else if (obj.type === 'tool_use') { - toolCalls += 1; - } else if (obj.type === 'result') { - const u = obj.usage ?? {}; - input += u.input_token_count ?? u.prompt_tokens ?? 0; - out += u.output_token_count ?? u.completion_tokens ?? 0; - if (obj.model) modelUsed = obj.model; - } - } catch { - // skip malformed lines - } - } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; - } - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { return { output: '', diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 12456ec23..98c2f3e03 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { if (result.error) { throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`); } - // Gemini CLI occasionally returns empty output even on successful runs - // (model returned content the CLI parser missed, intermittent stream issues). - // We assert the adapter ran end-to-end without erroring and reports a non- - // empty token count instead of grepping the literal "ok" — that string - // assertion was too brittle for a smoke that's really about "did the - // adapter wire up and the run terminate successfully?" - expect(typeof result.output).toBe('string'); - // Gemini CLI sometimes returns 0 tokens in the result event (older responses); - // assert non-negative instead of strictly positive. - expect(result.tokens.input).toBeGreaterThanOrEqual(0); - expect(result.tokens.output).toBeGreaterThanOrEqual(0); + // Adapter must never report empty-success (#2159). After content/stats + // parsing, a healthy run has non-empty assistant text + token counts. + expect(result.output.trim().length).toBeGreaterThan(0); + expect(result.output.toLowerCase()).toContain('ok'); + expect(result.tokens.input).toBeGreaterThan(0); + expect(result.tokens.output).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); + expect(result.modelUsed.length).toBeGreaterThan(0); }, 150_000); test('timeout error surfaces as error.code=timeout (no exception)', async () => {