diff --git a/test/helpers/gemini-session-runner.ts b/test/helpers/gemini-session-runner.ts index 4f2f040e3..3b58c79ca 100644 --- a/test/helpers/gemini-session-runner.ts +++ b/test/helpers/gemini-session-runner.ts @@ -8,7 +8,8 @@ * Key differences from Codex session-runner: * - Uses `gemini -p` instead of `codex exec` * - Output is NDJSON with event types: init, message, tool_use, tool_result, result - * - Uses `--output-format stream-json --yolo` instead of `--json -s read-only` + * - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only` + * (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs) * - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd * - Message events are streamed with `delta: true` — must concatenate */ @@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: { } // Build gemini command - const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo']; + // --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json; + // without it gemini exits FatalUntrustedWorkspaceError before any model call. + const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; // Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted), // cwd for skill discovery. Hermetic scrub with gemini's auth surface diff --git a/test/helpers/providers/gemini.test.ts b/test/helpers/providers/gemini.test.ts new file mode 100644 index 000000000..a0f404e29 --- /dev/null +++ b/test/helpers/providers/gemini.test.ts @@ -0,0 +1,138 @@ +import { describe, test, expect } from 'bun:test'; +import { parseGeminiStreamJson, resultFromGeminiStream } from './gemini'; + +// Current CLI shape (gemini ≥ ~0.11 / stream-json): content + role, tokens under result.stats +// Documented at https://geminicli.com/docs/cli/headless/ and gemini-cli PR #10883. +const CURRENT_CLI_FIXTURE = [ + '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}', + '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}', + '{"type":"message","role":"assistant","content":"PONG","delta":true}', + '{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}', +].join('\n'); + +// Legacy shape: text field, tokens under result.usage +const LEGACY_FIXTURE = [ + '{"type":"message","text":"hello"}', + '{"type":"tool_use","name":"run_shell_command"}', + '{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}', +].join('\n'); + +describe('parseGeminiStreamJson', () => { + test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => { + const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE); + expect(parsed.output).toBe('PONG'); + expect(parsed.tokens.input).toBe(24946); + expect(parsed.tokens.output).toBe(32); + expect(parsed.modelUsed).toBe('gemini-2.5-pro'); + expect(parsed.toolCalls).toBe(0); + }); + + test('current CLI: user role content must not concat into output', () => { + const raw = [ + '{"type":"message","role":"user","content":"PROMPT ECHO"}', + '{"type":"message","role":"assistant","content":"ok","delta":true}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('ok'); + expect(parsed.output).not.toContain('PROMPT ECHO'); + }); + + test('legacy: still accepts text + usage.input_token_count', () => { + const parsed = parseGeminiStreamJson(LEGACY_FIXTURE); + expect(parsed.output).toBe('hello'); + expect(parsed.tokens.input).toBe(100); + expect(parsed.tokens.output).toBe(5); + expect(parsed.toolCalls).toBe(1); + expect(parsed.modelUsed).toBe('gemini-2.5-flash'); + }); + + test('legacy text with role:user is ignored', () => { + const raw = [ + '{"type":"message","role":"user","text":"echo"}', + '{"type":"message","role":"assistant","text":"kept"}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('kept'); + }); + + test('concatenates multiple assistant content deltas', () => { + const raw = [ + '{"type":"message","role":"assistant","content":"A","delta":true}', + '{"type":"message","role":"assistant","content":"B","delta":true}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).output).toBe('AB'); + }); + + test('skips malformed lines without throwing', () => { + const raw = [ + '{"type":"init","model":"m1"}', + 'not json', + '{"type":"message","role":"assistant","content":"x","delta":true}', + '{incomplete', + '{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('x'); + expect(parsed.tokens).toEqual({ input: 1, output: 2 }); + expect(parsed.modelUsed).toBe('m1'); + }); + + test('empty / whitespace-only input yields empty parse (no throw)', () => { + const parsed = parseGeminiStreamJson(''); + expect(parsed.output).toBe(''); + expect(parsed.tokens).toEqual({ input: 0, output: 0 }); + expect(parsed.toolCalls).toBe(0); + expect(parsed.modelUsed).toBeUndefined(); + }); + + test('result.model overrides init.model when both present', () => { + const raw = [ + '{"type":"init","model":"from-init"}', + '{"type":"message","role":"assistant","content":"hi"}', + '{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result'); + }); +}); + +describe('resultFromGeminiStream (adapter post-CLI e2e)', () => { + test('current CLI fixture → success row with PONG + tokens (not $0)', () => { + const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE, { durationMs: 42 }); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('PONG'); + expect(result.tokens.input).toBe(24946); + expect(result.tokens.output).toBe(32); + expect(result.modelUsed).toBe('gemini-2.5-pro'); + expect(result.durationMs).toBe(42); + // Cost signal: non-zero tokens means estimateCost will not be $0. + expect(result.tokens.input + result.tokens.output).toBeGreaterThan(0); + }); + + test('legacy text-only success still works', () => { + const result = resultFromGeminiStream(LEGACY_FIXTURE, { durationMs: 10 }); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('hello'); + expect(result.toolCalls).toBe(1); + }); + + test('empty exit-0 stream → error row, never silent $0 success (#2159)', () => { + const emptySuccess = [ + '{"type":"init","model":"gemini-2.5-pro"}', + '{"type":"message","role":"user","content":"hi"}', + '{"type":"result","status":"success","stats":{"input_tokens":0,"output_tokens":0}}', + ].join('\n'); + const result = resultFromGeminiStream(emptySuccess, { durationMs: 5 }); + expect(result.error).toBeDefined(); + expect(result.error!.code).toBe('unknown'); + expect(result.error!.reason).toContain('empty output'); + expect(result.output).toBe(''); + expect(result.modelUsed).toBe('gemini-2.5-pro'); + }); + + test('pre-fix bug shape (text field missing, only content) would have been empty — now succeeds', () => { + // This is the exact failure mode from #2159: content present, no text. + const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('PONG'); + }); +}); diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 5e7abba13..b53d725d1 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -5,13 +5,108 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +export type GeminiStreamParse = { + output: string; + tokens: { input: number; output: number }; + toolCalls: number; + modelUsed?: string; +}; + +/** + * Parse gemini NDJSON stream events (exported for unit tests). + * + * Current CLI (`--output-format stream-json`) emits: + * init → model + * message { role, content, delta? } → concat assistant content + * tool_use → increment toolCalls + * result { stats: { input_tokens, output_tokens } } → tokens + * + * Legacy shape (still accepted): + * message { text } → concat text + * result { usage: { input_token_count, output_token_count } } → tokens + */ +export function parseGeminiStreamJson(raw: string): GeminiStreamParse { + let output = ''; + let input = 0; + let out = 0; + let toolCalls = 0; + let modelUsed: string | undefined; + for (const line of raw.split('\n')) { + const s = line.trim(); + if (!s) continue; + try { + const obj = JSON.parse(s); + if (obj.type === 'init') { + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } else if (obj.type === 'message') { + // Current CLI: content + role. Role guard is required — the CLI echoes + // the user prompt as role:'user', which must not land in output. + if (obj.role === 'assistant' && typeof obj.content === 'string') { + output += obj.content; + } else if (typeof obj.text === 'string' && obj.role !== 'user') { + // Legacy text field (no role, or assistant). + output += obj.text; + } + } else if (obj.type === 'tool_use') { + toolCalls += 1; + } else if (obj.type === 'result') { + const u = obj.usage ?? obj.stats ?? {}; + input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0; + out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0; + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } + } catch { + // skip malformed lines + } + } + return { output, tokens: { input, output: out }, toolCalls, modelUsed }; +} + +/** + * Map a raw stream-json dump to a RunResult, including the empty-success + * hardening from #2159. Exported so adapter e2e can exercise the full + * post-CLI path without a live gemini binary. + */ +export function resultFromGeminiStream( + raw: string, + opts: { model?: string; durationMs?: number } = {}, +): RunResult { + const parsed = parseGeminiStreamJson(raw); + const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro'; + const durationMs = opts.durationMs ?? 0; + if (!parsed.output.trim()) { + return { + output: '', + tokens: { input: 0, output: 0 }, + durationMs, + toolCalls: 0, + modelUsed, + error: { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' }, + }; + } + return { + output: parsed.output, + tokens: parsed.tokens, + durationMs, + toolCalls: parsed.toolCalls, + modelUsed, + }; +} + /** * Gemini adapter — wraps the `gemini` CLI. * - * Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output - * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format - * stream-json` is requested. This adapter uses a single-response form for simplicity - * in benchmarks; richer streaming lives in gemini-session-runner.ts. + * Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth. + * Personal OAuth free-tier is no longer supported by gemini CLI — use an + * AI Studio API key. Antigravity is a separate product/quota path. + * + * Headless flags always passed: + * --output-format stream-json — NDJSON events (message/tool_use/result) + * --yolo — auto-approve tools (non-interactive) + * --skip-trust — trust cwd for this session; required when + * workdir is a temp/untrusted folder (benchmarks + * use mkdtemp). Without it headless gemini exits + * before calling the model. */ export class GeminiAdapter implements ProviderAdapter { readonly name = 'gemini'; @@ -26,9 +121,14 @@ export class GeminiAdapter implements ProviderAdapter { const newCfgDir = path.join(os.homedir(), '.gemini'); const newOauth = path.join(newCfgDir, 'oauth_creds.json'); const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth); - const hasKey = !!process.env.GOOGLE_API_KEY; + // CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY. + const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY); if (!hasCfg && !hasKey) { - return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' }; + return { + ok: false, + reason: + 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.', + }; } return { ok: true }; } @@ -36,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); // Default to --yolo (non-interactive) and stream-json output so we can parse - // tokens + tool calls. Callers can override via extraArgs. - const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo']; + // tokens + tool calls. --skip-trust is required for headless/temp workdirs + // (gemini CLI otherwise exits: "not running in a trusted directory"). + // Callers can override via extraArgs. + const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs); @@ -47,15 +149,15 @@ export class GeminiAdapter implements ProviderAdapter { timeout: opts.timeoutMs, encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, + env: { + ...process.env, + // Prefer GEMINI_API_KEY when only that is set (CLI reads both). + ...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY + ? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY } + : {}), + }, }); - const parsed = this.parseStreamJson(out); - return { - output: parsed.output, - tokens: parsed.tokens, - durationMs: Date.now() - start, - toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro', - }; + return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start }); } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; @@ -63,7 +165,7 @@ export class GeminiAdapter implements ProviderAdapter { if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { + if (/unauthorized|auth|login|api key|ineligibletier|no longer supported/i.test(stderr)) { return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); } if (/rate[- ]?limit|429|quota/i.test(stderr)) { @@ -77,41 +179,6 @@ export class GeminiAdapter implements ProviderAdapter { return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro'); } - /** - * Parse gemini NDJSON stream events: - * init → session id (discarded here) - * message { delta: true, text } → concat to output - * tool_use { name } → increment toolCalls - * result { usage: { input_token_count, output_token_count } } → tokens - */ - private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { - let output = ''; - let input = 0; - let out = 0; - let toolCalls = 0; - let modelUsed: string | undefined; - for (const line of raw.split('\n')) { - const s = line.trim(); - if (!s) continue; - try { - const obj = JSON.parse(s); - if (obj.type === 'message' && typeof obj.text === 'string') { - output += obj.text; - } else if (obj.type === 'tool_use') { - toolCalls += 1; - } else if (obj.type === 'result') { - const u = obj.usage ?? {}; - input += u.input_token_count ?? u.prompt_tokens ?? 0; - out += u.output_token_count ?? u.completion_tokens ?? 0; - if (obj.model) modelUsed = obj.model; - } - } catch { - // skip malformed lines - } - } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; - } - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { return { output: '', diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 12456ec23..98c2f3e03 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { if (result.error) { throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`); } - // Gemini CLI occasionally returns empty output even on successful runs - // (model returned content the CLI parser missed, intermittent stream issues). - // We assert the adapter ran end-to-end without erroring and reports a non- - // empty token count instead of grepping the literal "ok" — that string - // assertion was too brittle for a smoke that's really about "did the - // adapter wire up and the run terminate successfully?" - expect(typeof result.output).toBe('string'); - // Gemini CLI sometimes returns 0 tokens in the result event (older responses); - // assert non-negative instead of strictly positive. - expect(result.tokens.input).toBeGreaterThanOrEqual(0); - expect(result.tokens.output).toBeGreaterThanOrEqual(0); + // Adapter must never report empty-success (#2159). After content/stats + // parsing, a healthy run has non-empty assistant text + token counts. + expect(result.output.trim().length).toBeGreaterThan(0); + expect(result.output.toLowerCase()).toContain('ok'); + expect(result.tokens.input).toBeGreaterThan(0); + expect(result.tokens.output).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); + expect(result.modelUsed.length).toBeGreaterThan(0); }, 150_000); test('timeout error surfaces as error.code=timeout (no exception)', async () => {