From 58e9881d6e9b684e438c1077976f75912f7e59b4 Mon Sep 17 00:00:00 2001 From: Sina Date: Fri, 10 Jul 2026 09:28:48 -0700 Subject: [PATCH 1/4] fix(benchmark): parse current gemini stream-json content/stats shape GeminiAdapter was reading message.text and result.usage, so current CLI content/stats events produced empty $0 success rows. Accept content with an assistant role guard, stats token fallbacks, init model, and treat empty exit-0 output as an error (#2159). Co-authored-by: Cursor --- test/helpers/providers/gemini.test.ts | 95 ++++++++++++++++++ test/helpers/providers/gemini.ts | 106 ++++++++++++++------- test/skill-e2e-benchmark-providers.test.ts | 18 ++-- 3 files changed, 171 insertions(+), 48 deletions(-) create mode 100644 test/helpers/providers/gemini.test.ts diff --git a/test/helpers/providers/gemini.test.ts b/test/helpers/providers/gemini.test.ts new file mode 100644 index 000000000..421e00c64 --- /dev/null +++ b/test/helpers/providers/gemini.test.ts @@ -0,0 +1,95 @@ +import { describe, test, expect } from 'bun:test'; +import { parseGeminiStreamJson } from './gemini'; + +// Current CLI shape (gemini ≥ ~1.58): content + role, tokens under result.stats +const CURRENT_CLI_FIXTURE = [ + '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}', + '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}', + '{"type":"message","role":"assistant","content":"PONG","delta":true}', + '{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}', +].join('\n'); + +// Legacy shape: text field, tokens under result.usage +const LEGACY_FIXTURE = [ + '{"type":"message","text":"hello"}', + '{"type":"tool_use","name":"run_shell_command"}', + '{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}', +].join('\n'); + +describe('parseGeminiStreamJson', () => { + test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => { + const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE); + expect(parsed.output).toBe('PONG'); + expect(parsed.tokens.input).toBe(24946); + expect(parsed.tokens.output).toBe(32); + expect(parsed.modelUsed).toBe('gemini-2.5-pro'); + expect(parsed.toolCalls).toBe(0); + }); + + test('current CLI: user role content must not concat into output', () => { + const raw = [ + '{"type":"message","role":"user","content":"PROMPT ECHO"}', + '{"type":"message","role":"assistant","content":"ok","delta":true}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('ok'); + expect(parsed.output).not.toContain('PROMPT ECHO'); + }); + + test('legacy: still accepts text + usage.input_token_count', () => { + const parsed = parseGeminiStreamJson(LEGACY_FIXTURE); + expect(parsed.output).toBe('hello'); + expect(parsed.tokens.input).toBe(100); + expect(parsed.tokens.output).toBe(5); + expect(parsed.toolCalls).toBe(1); + expect(parsed.modelUsed).toBe('gemini-2.5-flash'); + }); + + test('legacy text with role:user is ignored', () => { + const raw = [ + '{"type":"message","role":"user","text":"echo"}', + '{"type":"message","role":"assistant","text":"kept"}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('kept'); + }); + + test('concatenates multiple assistant content deltas', () => { + const raw = [ + '{"type":"message","role":"assistant","content":"A","delta":true}', + '{"type":"message","role":"assistant","content":"B","delta":true}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).output).toBe('AB'); + }); + + test('skips malformed lines without throwing', () => { + const raw = [ + '{"type":"init","model":"m1"}', + 'not json', + '{"type":"message","role":"assistant","content":"x","delta":true}', + '{incomplete', + '{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}', + ].join('\n'); + const parsed = parseGeminiStreamJson(raw); + expect(parsed.output).toBe('x'); + expect(parsed.tokens).toEqual({ input: 1, output: 2 }); + expect(parsed.modelUsed).toBe('m1'); + }); + + test('empty / whitespace-only input yields empty parse (no throw)', () => { + const parsed = parseGeminiStreamJson(''); + expect(parsed.output).toBe(''); + expect(parsed.tokens).toEqual({ input: 0, output: 0 }); + expect(parsed.toolCalls).toBe(0); + expect(parsed.modelUsed).toBeUndefined(); + }); + + test('result.model overrides init.model when both present', () => { + const raw = [ + '{"type":"init","model":"from-init"}', + '{"type":"message","role":"assistant","content":"hi"}', + '{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}', + ].join('\n'); + expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result'); + }); +}); diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 5e7abba13..cf3ad333b 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -5,6 +5,63 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +export type GeminiStreamParse = { + output: string; + tokens: { input: number; output: number }; + toolCalls: number; + modelUsed?: string; +}; + +/** + * Parse gemini NDJSON stream events (exported for unit tests). + * + * Current CLI (`--output-format stream-json`) emits: + * init → model + * message { role, content, delta? } → concat assistant content + * tool_use → increment toolCalls + * result { stats: { input_tokens, output_tokens } } → tokens + * + * Legacy shape (still accepted): + * message { text } → concat text + * result { usage: { input_token_count, output_token_count } } → tokens + */ +export function parseGeminiStreamJson(raw: string): GeminiStreamParse { + let output = ''; + let input = 0; + let out = 0; + let toolCalls = 0; + let modelUsed: string | undefined; + for (const line of raw.split('\n')) { + const s = line.trim(); + if (!s) continue; + try { + const obj = JSON.parse(s); + if (obj.type === 'init') { + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } else if (obj.type === 'message') { + // Current CLI: content + role. Role guard is required — the CLI echoes + // the user prompt as role:'user', which must not land in output. + if (obj.role === 'assistant' && typeof obj.content === 'string') { + output += obj.content; + } else if (typeof obj.text === 'string' && obj.role !== 'user') { + // Legacy text field (no role, or assistant). + output += obj.text; + } + } else if (obj.type === 'tool_use') { + toolCalls += 1; + } else if (obj.type === 'result') { + const u = obj.usage ?? obj.stats ?? {}; + input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0; + out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0; + if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model; + } + } catch { + // skip malformed lines + } + } + return { output, tokens: { input, output: out }, toolCalls, modelUsed }; +} + /** * Gemini adapter — wraps the `gemini` CLI. * @@ -48,13 +105,23 @@ export class GeminiAdapter implements ProviderAdapter { encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, }); - const parsed = this.parseStreamJson(out); + const parsed = parseGeminiStreamJson(out); + const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro'; + // Empty-success is indistinguishable from a healthy cheap run in the + // comparison table — never report it as a $0 success row (#2159). + if (!parsed.output.trim()) { + return this.emptyResult( + Date.now() - start, + { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' }, + modelUsed, + ); + } return { output: parsed.output, tokens: parsed.tokens, durationMs: Date.now() - start, toolCalls: parsed.toolCalls, - modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro', + modelUsed, }; } catch (err: unknown) { const durationMs = Date.now() - start; @@ -77,41 +144,6 @@ export class GeminiAdapter implements ProviderAdapter { return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro'); } - /** - * Parse gemini NDJSON stream events: - * init → session id (discarded here) - * message { delta: true, text } → concat to output - * tool_use { name } → increment toolCalls - * result { usage: { input_token_count, output_token_count } } → tokens - */ - private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { - let output = ''; - let input = 0; - let out = 0; - let toolCalls = 0; - let modelUsed: string | undefined; - for (const line of raw.split('\n')) { - const s = line.trim(); - if (!s) continue; - try { - const obj = JSON.parse(s); - if (obj.type === 'message' && typeof obj.text === 'string') { - output += obj.text; - } else if (obj.type === 'tool_use') { - toolCalls += 1; - } else if (obj.type === 'result') { - const u = obj.usage ?? {}; - input += u.input_token_count ?? u.prompt_tokens ?? 0; - out += u.output_token_count ?? u.completion_tokens ?? 0; - if (obj.model) modelUsed = obj.model; - } - } catch { - // skip malformed lines - } - } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; - } - private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { return { output: '', diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 12456ec23..98c2f3e03 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { if (result.error) { throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`); } - // Gemini CLI occasionally returns empty output even on successful runs - // (model returned content the CLI parser missed, intermittent stream issues). - // We assert the adapter ran end-to-end without erroring and reports a non- - // empty token count instead of grepping the literal "ok" — that string - // assertion was too brittle for a smoke that's really about "did the - // adapter wire up and the run terminate successfully?" - expect(typeof result.output).toBe('string'); - // Gemini CLI sometimes returns 0 tokens in the result event (older responses); - // assert non-negative instead of strictly positive. - expect(result.tokens.input).toBeGreaterThanOrEqual(0); - expect(result.tokens.output).toBeGreaterThanOrEqual(0); + // Adapter must never report empty-success (#2159). After content/stats + // parsing, a healthy run has non-empty assistant text + token counts. + expect(result.output.trim().length).toBeGreaterThan(0); + expect(result.output.toLowerCase()).toContain('ok'); + expect(result.tokens.input).toBeGreaterThan(0); + expect(result.tokens.output).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); + expect(result.modelUsed.length).toBeGreaterThan(0); }, 150_000); test('timeout error surfaces as error.code=timeout (no exception)', async () => { From f155481b78b7f2c538daa77d47fdd404089d6ccd Mon Sep 17 00:00:00 2001 From: Sina Date: Fri, 10 Jul 2026 09:28:48 -0700 Subject: [PATCH 2/4] chore: bump version and changelog (v1.60.2.0) Co-authored-by: Cursor --- CHANGELOG.md | 18 ++++++++++++++++++ VERSION | 2 +- package.json | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 351e7cabd..e47d737ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [1.60.2.0] - 2026-07-10 + +## **Gemini model benchmarks no longer rank a silent empty run as a $0 win.** + +`gstack-model-benchmark --models gemini` was parsing an older stream-json shape (`message.text`, `result.usage`). Current gemini CLI emits `message.content` (with `role`) and `result.stats`, so the adapter returned empty output, 0 tokens, and no error — a fake success that quietly mis-ranked gemini in comparison tables. + +### What this means for you + +Gemini benchmark rows now show real assistant text and token counts again. If a run somehow still exits 0 with empty output, it surfaces as an error row instead of a $0 success. + +### Itemized changes + +#### Fixed + +- `test/helpers/providers/gemini.ts`: `parseGeminiStreamJson` accepts current CLI `content` + `role:'assistant'` (ignores user-prompt echoes), reads tokens from `result.stats` with legacy `usage` fallbacks, and takes `model` from `init`. Empty exit-0 output is reported as `error.code=unknown` instead of a silent success (#2159). +- `test/helpers/providers/gemini.test.ts`: unit coverage for current + legacy stream shapes, user-echo guard, and empty parse. +- `test/skill-e2e-benchmark-providers.test.ts`: live gemini smoke asserts non-empty output containing the probe string and positive token counts. + ## [1.60.1.0] - 2026-07-09 ## **The /autoplan dual-voice eval is back on the board, catching real regressions.** diff --git a/VERSION b/VERSION index c4190e004..762f17554 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.60.1.0 +1.60.2.0 diff --git a/package.json b/package.json index c5168d648..4a628012f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.60.1.0", + "version": "1.60.2.0", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", From 5d93ce278d301aa05559939272ae0f522270580b Mon Sep 17 00:00:00 2001 From: Sina Date: Fri, 10 Jul 2026 09:38:06 -0700 Subject: [PATCH 3/4] test(benchmark): full post-CLI gemini e2e + GEMINI_API_KEY auth path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercise resultFromGeminiStream against current stream-json fixtures (including empty-success hardening). Recognize GEMINI_API_KEY and map IneligibleTierError to auth — personal OAuth free-tier is no longer supported by gemini CLI. Co-authored-by: Cursor --- test/helpers/providers/gemini.test.ts | 47 +++++++++++++++++- test/helpers/providers/gemini.ts | 68 ++++++++++++++++++--------- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/test/helpers/providers/gemini.test.ts b/test/helpers/providers/gemini.test.ts index 421e00c64..a0f404e29 100644 --- a/test/helpers/providers/gemini.test.ts +++ b/test/helpers/providers/gemini.test.ts @@ -1,7 +1,8 @@ import { describe, test, expect } from 'bun:test'; -import { parseGeminiStreamJson } from './gemini'; +import { parseGeminiStreamJson, resultFromGeminiStream } from './gemini'; -// Current CLI shape (gemini ≥ ~1.58): content + role, tokens under result.stats +// Current CLI shape (gemini ≥ ~0.11 / stream-json): content + role, tokens under result.stats +// Documented at https://geminicli.com/docs/cli/headless/ and gemini-cli PR #10883. const CURRENT_CLI_FIXTURE = [ '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}', '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}', @@ -93,3 +94,45 @@ describe('parseGeminiStreamJson', () => { expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result'); }); }); + +describe('resultFromGeminiStream (adapter post-CLI e2e)', () => { + test('current CLI fixture → success row with PONG + tokens (not $0)', () => { + const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE, { durationMs: 42 }); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('PONG'); + expect(result.tokens.input).toBe(24946); + expect(result.tokens.output).toBe(32); + expect(result.modelUsed).toBe('gemini-2.5-pro'); + expect(result.durationMs).toBe(42); + // Cost signal: non-zero tokens means estimateCost will not be $0. + expect(result.tokens.input + result.tokens.output).toBeGreaterThan(0); + }); + + test('legacy text-only success still works', () => { + const result = resultFromGeminiStream(LEGACY_FIXTURE, { durationMs: 10 }); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('hello'); + expect(result.toolCalls).toBe(1); + }); + + test('empty exit-0 stream → error row, never silent $0 success (#2159)', () => { + const emptySuccess = [ + '{"type":"init","model":"gemini-2.5-pro"}', + '{"type":"message","role":"user","content":"hi"}', + '{"type":"result","status":"success","stats":{"input_tokens":0,"output_tokens":0}}', + ].join('\n'); + const result = resultFromGeminiStream(emptySuccess, { durationMs: 5 }); + expect(result.error).toBeDefined(); + expect(result.error!.code).toBe('unknown'); + expect(result.error!.reason).toContain('empty output'); + expect(result.output).toBe(''); + expect(result.modelUsed).toBe('gemini-2.5-pro'); + }); + + test('pre-fix bug shape (text field missing, only content) would have been empty — now succeeds', () => { + // This is the exact failure mode from #2159: content present, no text. + const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE); + expect(result.error).toBeUndefined(); + expect(result.output).toBe('PONG'); + }); +}); diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index cf3ad333b..84b3059ab 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -62,6 +62,37 @@ export function parseGeminiStreamJson(raw: string): GeminiStreamParse { return { output, tokens: { input, output: out }, toolCalls, modelUsed }; } +/** + * Map a raw stream-json dump to a RunResult, including the empty-success + * hardening from #2159. Exported so adapter e2e can exercise the full + * post-CLI path without a live gemini binary. + */ +export function resultFromGeminiStream( + raw: string, + opts: { model?: string; durationMs?: number } = {}, +): RunResult { + const parsed = parseGeminiStreamJson(raw); + const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro'; + const durationMs = opts.durationMs ?? 0; + if (!parsed.output.trim()) { + return { + output: '', + tokens: { input: 0, output: 0 }, + durationMs, + toolCalls: 0, + modelUsed, + error: { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' }, + }; + } + return { + output: parsed.output, + tokens: parsed.tokens, + durationMs, + toolCalls: parsed.toolCalls, + modelUsed, + }; +} + /** * Gemini adapter — wraps the `gemini` CLI. * @@ -83,9 +114,14 @@ export class GeminiAdapter implements ProviderAdapter { const newCfgDir = path.join(os.homedir(), '.gemini'); const newOauth = path.join(newCfgDir, 'oauth_creds.json'); const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth); - const hasKey = !!process.env.GOOGLE_API_KEY; + // CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY. + const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY); if (!hasCfg && !hasKey) { - return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' }; + return { + ok: false, + reason: + 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.', + }; } return { ok: true }; } @@ -104,25 +140,15 @@ export class GeminiAdapter implements ProviderAdapter { timeout: opts.timeoutMs, encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, + env: { + ...process.env, + // Prefer GEMINI_API_KEY when only that is set (CLI reads both). + ...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY + ? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY } + : {}), + }, }); - const parsed = parseGeminiStreamJson(out); - const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro'; - // Empty-success is indistinguishable from a healthy cheap run in the - // comparison table — never report it as a $0 success row (#2159). - if (!parsed.output.trim()) { - return this.emptyResult( - Date.now() - start, - { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' }, - modelUsed, - ); - } - return { - output: parsed.output, - tokens: parsed.tokens, - durationMs: Date.now() - start, - toolCalls: parsed.toolCalls, - modelUsed, - }; + return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start }); } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; @@ -130,7 +156,7 @@ export class GeminiAdapter implements ProviderAdapter { if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { + if (/unauthorized|auth|login|api key|ineligibletier|no longer supported/i.test(stderr)) { return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); } if (/rate[- ]?limit|429|quota/i.test(stderr)) { From c7e182baa20d5805b93f35c677bc60c9f657d656 Mon Sep 17 00:00:00 2001 From: Sina Date: Fri, 10 Jul 2026 10:38:46 -0700 Subject: [PATCH 4/4] fix(benchmark): skip-trust for headless gemini + drop release metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass --skip-trust so temp/untrusted workdirs work in headless benchmarks (mirrors Codex --skip-git-repo-check). Revert VERSION, CHANGELOG, and package.json bumps — leave release bookkeeping to maintainers / /ship, matching community PR practice. Co-authored-by: Cursor --- CHANGELOG.md | 18 ------------------ VERSION | 2 +- package.json | 2 +- test/helpers/gemini-session-runner.ts | 7 +++++-- test/helpers/providers/gemini.ts | 21 +++++++++++++++------ 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47d737ca..351e7cabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,5 @@ # Changelog -## [1.60.2.0] - 2026-07-10 - -## **Gemini model benchmarks no longer rank a silent empty run as a $0 win.** - -`gstack-model-benchmark --models gemini` was parsing an older stream-json shape (`message.text`, `result.usage`). Current gemini CLI emits `message.content` (with `role`) and `result.stats`, so the adapter returned empty output, 0 tokens, and no error — a fake success that quietly mis-ranked gemini in comparison tables. - -### What this means for you - -Gemini benchmark rows now show real assistant text and token counts again. If a run somehow still exits 0 with empty output, it surfaces as an error row instead of a $0 success. - -### Itemized changes - -#### Fixed - -- `test/helpers/providers/gemini.ts`: `parseGeminiStreamJson` accepts current CLI `content` + `role:'assistant'` (ignores user-prompt echoes), reads tokens from `result.stats` with legacy `usage` fallbacks, and takes `model` from `init`. Empty exit-0 output is reported as `error.code=unknown` instead of a silent success (#2159). -- `test/helpers/providers/gemini.test.ts`: unit coverage for current + legacy stream shapes, user-echo guard, and empty parse. -- `test/skill-e2e-benchmark-providers.test.ts`: live gemini smoke asserts non-empty output containing the probe string and positive token counts. - ## [1.60.1.0] - 2026-07-09 ## **The /autoplan dual-voice eval is back on the board, catching real regressions.** diff --git a/VERSION b/VERSION index 762f17554..c4190e004 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.60.2.0 +1.60.1.0 diff --git a/package.json b/package.json index 4a628012f..c5168d648 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.60.2.0", + "version": "1.60.1.0", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/helpers/gemini-session-runner.ts b/test/helpers/gemini-session-runner.ts index 4f2f040e3..3b58c79ca 100644 --- a/test/helpers/gemini-session-runner.ts +++ b/test/helpers/gemini-session-runner.ts @@ -8,7 +8,8 @@ * Key differences from Codex session-runner: * - Uses `gemini -p` instead of `codex exec` * - Output is NDJSON with event types: init, message, tool_use, tool_result, result - * - Uses `--output-format stream-json --yolo` instead of `--json -s read-only` + * - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only` + * (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs) * - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd * - Message events are streamed with `delta: true` — must concatenate */ @@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: { } // Build gemini command - const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo']; + // --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json; + // without it gemini exits FatalUntrustedWorkspaceError before any model call. + const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; // Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted), // cwd for skill discovery. Hermetic scrub with gemini's auth surface diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 84b3059ab..b53d725d1 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -96,10 +96,17 @@ export function resultFromGeminiStream( /** * Gemini adapter — wraps the `gemini` CLI. * - * Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output - * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format - * stream-json` is requested. This adapter uses a single-response form for simplicity - * in benchmarks; richer streaming lives in gemini-session-runner.ts. + * Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth. + * Personal OAuth free-tier is no longer supported by gemini CLI — use an + * AI Studio API key. Antigravity is a separate product/quota path. + * + * Headless flags always passed: + * --output-format stream-json — NDJSON events (message/tool_use/result) + * --yolo — auto-approve tools (non-interactive) + * --skip-trust — trust cwd for this session; required when + * workdir is a temp/untrusted folder (benchmarks + * use mkdtemp). Without it headless gemini exits + * before calling the model. */ export class GeminiAdapter implements ProviderAdapter { readonly name = 'gemini'; @@ -129,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); // Default to --yolo (non-interactive) and stream-json output so we can parse - // tokens + tool calls. Callers can override via extraArgs. - const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo']; + // tokens + tool calls. --skip-trust is required for headless/temp workdirs + // (gemini CLI otherwise exits: "not running in a trusted directory"). + // Callers can override via extraArgs. + const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs);