fix: detect GEMINI_API_KEY/.env and parse current Gemini stream schema

The benchmark adapter only recognized GOOGLE_API_KEY and parsed the legacy
text/usage event shape. Recognize GEMINI_API_KEY and ~/.gemini/.env, parse
the current content/stats schema, ignore echoed user messages, and run
report-only plan mode with --skip-trust for disposable workspaces.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/model-benchmark/providers/gemini.ts

@@ -8,7 +8,8 @@ import * as os from 'os';
 /**
  * Gemini adapter — wraps the `gemini` CLI.
  *
- * Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
+ * Gemini CLI auth comes from its OAuth files, ~/.gemini/.env, or the
+ * GOOGLE_API_KEY/GEMINI_API_KEY environment variables. Output
  * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
  * stream-json` is requested. This adapter uses a single-response form for simplicity
  * in benchmarks; richer streaming lives in gemini-session-runner.ts.
@@ -25,19 +26,22 @@ export class GeminiAdapter implements ProviderAdapter {
     const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
     const newCfgDir = path.join(os.homedir(), '.gemini');
     const newOauth = path.join(newCfgDir, 'oauth_creds.json');
+    const geminiEnv = path.join(newCfgDir, '.env');
     const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
-    const hasKey = !!process.env.GOOGLE_API_KEY;
+    const hasEnvFileKey = fs.existsSync(geminiEnv)
+      && /^(?:GOOGLE_API_KEY|GEMINI_API_KEY)\s*=/m.test(fs.readFileSync(geminiEnv, 'utf-8'));
+    const hasKey = !!process.env.GOOGLE_API_KEY || !!process.env.GEMINI_API_KEY || hasEnvFileKey;
     if (!hasCfg && !hasKey) {
-      return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
+      return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY/GEMINI_API_KEY.' };
     }
     return { ok: true };
   }
 
   async run(opts: RunOpts): Promise<RunResult> {
     const start = Date.now();
-    // Default to --yolo (non-interactive) and stream-json output so we can parse
+    // Benchmarks are report-only. Plan mode keeps the CLI non-interactive while
-    // tokens + tool calls. Callers can override via extraArgs.
+    // preventing a benchmark prompt from mutating the target worktree.
-    const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
+    const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--approval-mode', 'plan', '--skip-trust'];
     if (opts.model) args.push('--model', opts.model);
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
@@ -48,7 +52,7 @@ export class GeminiAdapter implements ProviderAdapter {
         encoding: 'utf-8',
         maxBuffer: 32 * 1024 * 1024,
       });
-      const parsed = this.parseStreamJson(out);
+      const parsed = parseGeminiStreamJson(out);
       return {
         output: parsed.output,
         tokens: parsed.tokens,
@@ -77,41 +81,6 @@ export class GeminiAdapter implements ProviderAdapter {
     return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
   }
 
-  /**
-   * Parse gemini NDJSON stream events:
-   *   init  → session id (discarded here)
-   *   message { delta: true, text } → concat to output
-   *   tool_use { name } → increment toolCalls
-   *   result { usage: { input_token_count, output_token_count } } → tokens
-   */
-  private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
-    let output = '';
-    let input = 0;
-    let out = 0;
-    let toolCalls = 0;
-    let modelUsed: string | undefined;
-    for (const line of raw.split('\n')) {
-      const s = line.trim();
-      if (!s) continue;
-      try {
-        const obj = JSON.parse(s);
-        if (obj.type === 'message' && typeof obj.text === 'string') {
-          output += obj.text;
-        } else if (obj.type === 'tool_use') {
-          toolCalls += 1;
-        } else if (obj.type === 'result') {
-          const u = obj.usage ?? {};
-          input += u.input_token_count ?? u.prompt_tokens ?? 0;
-          out += u.output_token_count ?? u.completion_tokens ?? 0;
-          if (obj.model) modelUsed = obj.model;
-        }
-      } catch {
-        // skip malformed lines
-      }
-    }
-    return { output, tokens: { input, output: out }, toolCalls, modelUsed };
-  }
-
   private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
     return {
       output: '',
@@ -123,3 +92,43 @@ export class GeminiAdapter implements ProviderAdapter {
     };
   }
 }
+
+/** Parse both legacy `usage` events and current Gemini CLI `stats` events. */
+export function parseGeminiStreamJson(raw: string): {
+  output: string;
+  tokens: { input: number; output: number };
+  toolCalls: number;
+  modelUsed?: string;
+} {
+    let output = '';
+    let input = 0;
+    let out = 0;
+    let toolCalls = 0;
+    let modelUsed: string | undefined;
+    for (const line of raw.split('\n')) {
+      const s = line.trim();
+      if (!s) continue;
+      try {
+        const obj = JSON.parse(s);
+        if (obj.type === 'init' && typeof obj.model === 'string' && obj.model !== 'auto') {
+          modelUsed = obj.model;
+        } else if (obj.type === 'message' && obj.role === 'assistant') {
+          const content = typeof obj.content === 'string' ? obj.content : obj.text;
+          if (typeof content === 'string') output += content;
+        } else if (obj.type === 'tool_use') {
+          toolCalls += 1;
+        } else if (obj.type === 'result') {
+          const u = obj.usage ?? obj.stats ?? {};
+          input += u.input_token_count ?? u.prompt_tokens ?? u.input_tokens ?? u.input ?? 0;
+          out += u.output_token_count ?? u.completion_tokens ?? u.output_tokens ?? 0;
+          if (obj.model) modelUsed = obj.model;
+          if (!modelUsed && u.models && typeof u.models === 'object') {
+            modelUsed = Object.keys(u.models)[0];
+          }
+        }
+      } catch {
+        // skip malformed lines
+      }
+    }
+  return { output, tokens: { input, output: out }, toolCalls, modelUsed };
+}

test/gemini-benchmark-adapter.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from 'bun:test';
+import { parseGeminiStreamJson } from '../lib/model-benchmark/providers/gemini';
+
+describe('parseGeminiStreamJson', () => {
+  test('parses the current Gemini CLI content/stats schema', () => {
+    const raw = [
+      JSON.stringify({ type: 'init', model: 'auto' }),
+      JSON.stringify({ type: 'message', role: 'user', content: 'ignore me' }),
+      JSON.stringify({ type: 'message', role: 'assistant', content: 'real output' }),
+      JSON.stringify({ type: 'tool_use', tool_name: 'read_file' }),
+      JSON.stringify({
+        type: 'result',
+        stats: {
+          input_tokens: 101,
+          output_tokens: 17,
+          models: { 'gemini-3.1-flash-lite': { api: { totalRequests: 1 } } },
+        },
+      }),
+    ].join('\n');
+
+    expect(parseGeminiStreamJson(raw)).toEqual({
+      output: 'real output',
+      tokens: { input: 101, output: 17 },
+      toolCalls: 1,
+      modelUsed: 'gemini-3.1-flash-lite',
+    });
+  });
+
+  test('retains compatibility with legacy text/usage fields', () => {
+    const raw = [
+      JSON.stringify({ type: 'init', model: 'gemini-2.5-pro' }),
+      JSON.stringify({ type: 'message', role: 'assistant', text: 'legacy output' }),
+      JSON.stringify({
+        type: 'result',
+        usage: { input_token_count: 23, output_token_count: 5 },
+      }),
+      '{malformed',
+    ].join('\n');
+
+    expect(parseGeminiStreamJson(raw)).toEqual({
+      output: 'legacy output',
+      tokens: { input: 23, output: 5 },
+      toolCalls: 0,
+      modelUsed: 'gemini-2.5-pro',
+    });
+  });
+});