Merge pull request #2221 from time-attack/fix-gstack-issue

fix(benchmark): gemini stream-json content/stats parse (#2159)

test/helpers/gemini-session-runner.ts

@@ -8,7 +8,8 @@
  * Key differences from Codex session-runner:
  * - Uses `gemini -p` instead of `codex exec`
  * - Output is NDJSON with event types: init, message, tool_use, tool_result, result
- * - Uses `--output-format stream-json --yolo` instead of `--json -s read-only`
+ * - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only`
+ *   (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs)
  * - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
  * - Message events are streamed with `delta: true` — must concatenate
  */
@@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: {
   }
 
   // Build gemini command
-  const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo'];
+  // --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json;
+  // without it gemini exits FatalUntrustedWorkspaceError before any model call.
+  const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
 
   // Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
   // cwd for skill discovery. Hermetic scrub with gemini's auth surface

test/helpers/providers/gemini.test.ts

@@ -0,0 +1,138 @@
+import { describe, test, expect } from 'bun:test';
+import { parseGeminiStreamJson, resultFromGeminiStream } from './gemini';
+
+// Current CLI shape (gemini ≥ ~0.11 / stream-json): content + role, tokens under result.stats
+// Documented at https://geminicli.com/docs/cli/headless/ and gemini-cli PR #10883.
+const CURRENT_CLI_FIXTURE = [
+  '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}',
+  '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}',
+  '{"type":"message","role":"assistant","content":"PONG","delta":true}',
+  '{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}',
+].join('\n');
+
+// Legacy shape: text field, tokens under result.usage
+const LEGACY_FIXTURE = [
+  '{"type":"message","text":"hello"}',
+  '{"type":"tool_use","name":"run_shell_command"}',
+  '{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}',
+].join('\n');
+
+describe('parseGeminiStreamJson', () => {
+  test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => {
+    const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE);
+    expect(parsed.output).toBe('PONG');
+    expect(parsed.tokens.input).toBe(24946);
+    expect(parsed.tokens.output).toBe(32);
+    expect(parsed.modelUsed).toBe('gemini-2.5-pro');
+    expect(parsed.toolCalls).toBe(0);
+  });
+
+  test('current CLI: user role content must not concat into output', () => {
+    const raw = [
+      '{"type":"message","role":"user","content":"PROMPT ECHO"}',
+      '{"type":"message","role":"assistant","content":"ok","delta":true}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('ok');
+    expect(parsed.output).not.toContain('PROMPT ECHO');
+  });
+
+  test('legacy: still accepts text + usage.input_token_count', () => {
+    const parsed = parseGeminiStreamJson(LEGACY_FIXTURE);
+    expect(parsed.output).toBe('hello');
+    expect(parsed.tokens.input).toBe(100);
+    expect(parsed.tokens.output).toBe(5);
+    expect(parsed.toolCalls).toBe(1);
+    expect(parsed.modelUsed).toBe('gemini-2.5-flash');
+  });
+
+  test('legacy text with role:user is ignored', () => {
+    const raw = [
+      '{"type":"message","role":"user","text":"echo"}',
+      '{"type":"message","role":"assistant","text":"kept"}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('kept');
+  });
+
+  test('concatenates multiple assistant content deltas', () => {
+    const raw = [
+      '{"type":"message","role":"assistant","content":"A","delta":true}',
+      '{"type":"message","role":"assistant","content":"B","delta":true}',
+    ].join('\n');
+    expect(parseGeminiStreamJson(raw).output).toBe('AB');
+  });
+
+  test('skips malformed lines without throwing', () => {
+    const raw = [
+      '{"type":"init","model":"m1"}',
+      'not json',
+      '{"type":"message","role":"assistant","content":"x","delta":true}',
+      '{incomplete',
+      '{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('x');
+    expect(parsed.tokens).toEqual({ input: 1, output: 2 });
+    expect(parsed.modelUsed).toBe('m1');
+  });
+
+  test('empty / whitespace-only input yields empty parse (no throw)', () => {
+    const parsed = parseGeminiStreamJson('');
+    expect(parsed.output).toBe('');
+    expect(parsed.tokens).toEqual({ input: 0, output: 0 });
+    expect(parsed.toolCalls).toBe(0);
+    expect(parsed.modelUsed).toBeUndefined();
+  });
+
+  test('result.model overrides init.model when both present', () => {
+    const raw = [
+      '{"type":"init","model":"from-init"}',
+      '{"type":"message","role":"assistant","content":"hi"}',
+      '{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}',
+    ].join('\n');
+    expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result');
+  });
+});
+
+describe('resultFromGeminiStream (adapter post-CLI e2e)', () => {
+  test('current CLI fixture → success row with PONG + tokens (not $0)', () => {
+    const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE, { durationMs: 42 });
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('PONG');
+    expect(result.tokens.input).toBe(24946);
+    expect(result.tokens.output).toBe(32);
+    expect(result.modelUsed).toBe('gemini-2.5-pro');
+    expect(result.durationMs).toBe(42);
+    // Cost signal: non-zero tokens means estimateCost will not be $0.
+    expect(result.tokens.input + result.tokens.output).toBeGreaterThan(0);
+  });
+
+  test('legacy text-only success still works', () => {
+    const result = resultFromGeminiStream(LEGACY_FIXTURE, { durationMs: 10 });
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('hello');
+    expect(result.toolCalls).toBe(1);
+  });
+
+  test('empty exit-0 stream → error row, never silent $0 success (#2159)', () => {
+    const emptySuccess = [
+      '{"type":"init","model":"gemini-2.5-pro"}',
+      '{"type":"message","role":"user","content":"hi"}',
+      '{"type":"result","status":"success","stats":{"input_tokens":0,"output_tokens":0}}',
+    ].join('\n');
+    const result = resultFromGeminiStream(emptySuccess, { durationMs: 5 });
+    expect(result.error).toBeDefined();
+    expect(result.error!.code).toBe('unknown');
+    expect(result.error!.reason).toContain('empty output');
+    expect(result.output).toBe('');
+    expect(result.modelUsed).toBe('gemini-2.5-pro');
+  });
+
+  test('pre-fix bug shape (text field missing, only content) would have been empty — now succeeds', () => {
+    // This is the exact failure mode from #2159: content present, no text.
+    const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE);
+    expect(result.error).toBeUndefined();
+    expect(result.output).toBe('PONG');
+  });
+});

test/helpers/providers/gemini.ts

@@ -5,13 +5,108 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 
+export type GeminiStreamParse = {
+  output: string;
+  tokens: { input: number; output: number };
+  toolCalls: number;
+  modelUsed?: string;
+};
+
+/**
+ * Parse gemini NDJSON stream events (exported for unit tests).
+ *
+ * Current CLI (`--output-format stream-json`) emits:
+ *   init  → model
+ *   message { role, content, delta? } → concat assistant content
+ *   tool_use → increment toolCalls
+ *   result { stats: { input_tokens, output_tokens } } → tokens
+ *
+ * Legacy shape (still accepted):
+ *   message { text } → concat text
+ *   result { usage: { input_token_count, output_token_count } } → tokens
+ */
+export function parseGeminiStreamJson(raw: string): GeminiStreamParse {
+  let output = '';
+  let input = 0;
+  let out = 0;
+  let toolCalls = 0;
+  let modelUsed: string | undefined;
+  for (const line of raw.split('\n')) {
+    const s = line.trim();
+    if (!s) continue;
+    try {
+      const obj = JSON.parse(s);
+      if (obj.type === 'init') {
+        if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
+      } else if (obj.type === 'message') {
+        // Current CLI: content + role. Role guard is required — the CLI echoes
+        // the user prompt as role:'user', which must not land in output.
+        if (obj.role === 'assistant' && typeof obj.content === 'string') {
+          output += obj.content;
+        } else if (typeof obj.text === 'string' && obj.role !== 'user') {
+          // Legacy text field (no role, or assistant).
+          output += obj.text;
+        }
+      } else if (obj.type === 'tool_use') {
+        toolCalls += 1;
+      } else if (obj.type === 'result') {
+        const u = obj.usage ?? obj.stats ?? {};
+        input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0;
+        out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0;
+        if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
+      }
+    } catch {
+      // skip malformed lines
+    }
+  }
+  return { output, tokens: { input, output: out }, toolCalls, modelUsed };
+}
+
+/**
+ * Map a raw stream-json dump to a RunResult, including the empty-success
+ * hardening from #2159. Exported so adapter e2e can exercise the full
+ * post-CLI path without a live gemini binary.
+ */
+export function resultFromGeminiStream(
+  raw: string,
+  opts: { model?: string; durationMs?: number } = {},
+): RunResult {
+  const parsed = parseGeminiStreamJson(raw);
+  const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro';
+  const durationMs = opts.durationMs ?? 0;
+  if (!parsed.output.trim()) {
+    return {
+      output: '',
+      tokens: { input: 0, output: 0 },
+      durationMs,
+      toolCalls: 0,
+      modelUsed,
+      error: { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' },
+    };
+  }
+  return {
+    output: parsed.output,
+    tokens: parsed.tokens,
+    durationMs,
+    toolCalls: parsed.toolCalls,
+    modelUsed,
+  };
+}
+
 /**
  * Gemini adapter — wraps the `gemini` CLI.
  *
- * Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
+ * Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
- * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
+ *   Personal OAuth free-tier is no longer supported by gemini CLI — use an
- * stream-json` is requested. This adapter uses a single-response form for simplicity
+ *   AI Studio API key. Antigravity is a separate product/quota path.
- * in benchmarks; richer streaming lives in gemini-session-runner.ts.
+ *
+ * Headless flags always passed:
+ *   --output-format stream-json  — NDJSON events (message/tool_use/result)
+ *   --yolo                       — auto-approve tools (non-interactive)
+ *   --skip-trust                 — trust cwd for this session; required when
+ *                                  workdir is a temp/untrusted folder (benchmarks
+ *                                  use mkdtemp). Without it headless gemini exits
+ *                                  before calling the model.
  */
 export class GeminiAdapter implements ProviderAdapter {
   readonly name = 'gemini';
@@ -26,9 +121,14 @@ export class GeminiAdapter implements ProviderAdapter {
     const newCfgDir = path.join(os.homedir(), '.gemini');
     const newOauth = path.join(newCfgDir, 'oauth_creds.json');
     const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
-    const hasKey = !!process.env.GOOGLE_API_KEY;
+    // CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
+    const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
     if (!hasCfg && !hasKey) {
-      return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
+      return {
+        ok: false,
+        reason:
+          'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
+      };
     }
     return { ok: true };
   }
@@ -36,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter {
   async run(opts: RunOpts): Promise<RunResult> {
     const start = Date.now();
     // Default to --yolo (non-interactive) and stream-json output so we can parse
-    // tokens + tool calls. Callers can override via extraArgs.
+    // tokens + tool calls. --skip-trust is required for headless/temp workdirs
-    const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
+    // (gemini CLI otherwise exits: "not running in a trusted directory").
+    // Callers can override via extraArgs.
+    const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
     if (opts.model) args.push('--model', opts.model);
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
@@ -47,15 +149,15 @@ export class GeminiAdapter implements ProviderAdapter {
         timeout: opts.timeoutMs,
         encoding: 'utf-8',
         maxBuffer: 32 * 1024 * 1024,
+        env: {
+          ...process.env,
+          // Prefer GEMINI_API_KEY when only that is set (CLI reads both).
+          ...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY
+            ? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY }
+            : {}),
+        },
       });
-      const parsed = this.parseStreamJson(out);
+      return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start });
-      return {
-        output: parsed.output,
-        tokens: parsed.tokens,
-        durationMs: Date.now() - start,
-        toolCalls: parsed.toolCalls,
-        modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
-      };
     } catch (err: unknown) {
       const durationMs = Date.now() - start;
       const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -63,7 +165,7 @@ export class GeminiAdapter implements ProviderAdapter {
       if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
         return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
       }
-      if (/unauthorized|auth|login|api key/i.test(stderr)) {
+      if (/unauthorized|auth|login|api key|ineligibletier|no longer supported/i.test(stderr)) {
         return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
       }
       if (/rate[- ]?limit|429|quota/i.test(stderr)) {
@@ -77,41 +179,6 @@ export class GeminiAdapter implements ProviderAdapter {
     return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
   }
 
-  /**
-   * Parse gemini NDJSON stream events:
-   *   init  → session id (discarded here)
-   *   message { delta: true, text } → concat to output
-   *   tool_use { name } → increment toolCalls
-   *   result { usage: { input_token_count, output_token_count } } → tokens
-   */
-  private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
-    let output = '';
-    let input = 0;
-    let out = 0;
-    let toolCalls = 0;
-    let modelUsed: string | undefined;
-    for (const line of raw.split('\n')) {
-      const s = line.trim();
-      if (!s) continue;
-      try {
-        const obj = JSON.parse(s);
-        if (obj.type === 'message' && typeof obj.text === 'string') {
-          output += obj.text;
-        } else if (obj.type === 'tool_use') {
-          toolCalls += 1;
-        } else if (obj.type === 'result') {
-          const u = obj.usage ?? {};
-          input += u.input_token_count ?? u.prompt_tokens ?? 0;
-          out += u.output_token_count ?? u.completion_tokens ?? 0;
-          if (obj.model) modelUsed = obj.model;
-        }
-      } catch {
-        // skip malformed lines
-      }
-    }
-    return { output, tokens: { input, output: out }, toolCalls, modelUsed };
-  }
-
   private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
     return {
       output: '',

test/skill-e2e-benchmark-providers.test.ts

@@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     if (result.error) {
       throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`);
     }
-    // Gemini CLI occasionally returns empty output even on successful runs
+    // Adapter must never report empty-success (#2159). After content/stats
-    // (model returned content the CLI parser missed, intermittent stream issues).
+    // parsing, a healthy run has non-empty assistant text + token counts.
-    // We assert the adapter ran end-to-end without erroring and reports a non-
+    expect(result.output.trim().length).toBeGreaterThan(0);
-    // empty token count instead of grepping the literal "ok" — that string
+    expect(result.output.toLowerCase()).toContain('ok');
-    // assertion was too brittle for a smoke that's really about "did the
+    expect(result.tokens.input).toBeGreaterThan(0);
-    // adapter wire up and the run terminate successfully?"
+    expect(result.tokens.output).toBeGreaterThan(0);
-    expect(typeof result.output).toBe('string');
-    // Gemini CLI sometimes returns 0 tokens in the result event (older responses);
-    // assert non-negative instead of strictly positive.
-    expect(result.tokens.input).toBeGreaterThanOrEqual(0);
-    expect(result.tokens.output).toBeGreaterThanOrEqual(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
+    expect(result.modelUsed.length).toBeGreaterThan(0);
   }, 150_000);
 
   test('timeout error surfaces as error.code=timeout (no exception)', async () => {