diff --git a/CHANGELOG.md b/CHANGELOG.md index e47d737ca..351e7cabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,5 @@ # Changelog -## [1.60.2.0] - 2026-07-10 - -## **Gemini model benchmarks no longer rank a silent empty run as a $0 win.** - -`gstack-model-benchmark --models gemini` was parsing an older stream-json shape (`message.text`, `result.usage`). Current gemini CLI emits `message.content` (with `role`) and `result.stats`, so the adapter returned empty output, 0 tokens, and no error — a fake success that quietly mis-ranked gemini in comparison tables. - -### What this means for you - -Gemini benchmark rows now show real assistant text and token counts again. If a run somehow still exits 0 with empty output, it surfaces as an error row instead of a $0 success. - -### Itemized changes - -#### Fixed - -- `test/helpers/providers/gemini.ts`: `parseGeminiStreamJson` accepts current CLI `content` + `role:'assistant'` (ignores user-prompt echoes), reads tokens from `result.stats` with legacy `usage` fallbacks, and takes `model` from `init`. Empty exit-0 output is reported as `error.code=unknown` instead of a silent success (#2159). -- `test/helpers/providers/gemini.test.ts`: unit coverage for current + legacy stream shapes, user-echo guard, and empty parse. -- `test/skill-e2e-benchmark-providers.test.ts`: live gemini smoke asserts non-empty output containing the probe string and positive token counts. - ## [1.60.1.0] - 2026-07-09 ## **The /autoplan dual-voice eval is back on the board, catching real regressions.** diff --git a/VERSION b/VERSION index 762f17554..c4190e004 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.60.2.0 +1.60.1.0 diff --git a/package.json b/package.json index 4a628012f..c5168d648 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.60.2.0", + "version": "1.60.1.0", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/helpers/gemini-session-runner.ts b/test/helpers/gemini-session-runner.ts index 4f2f040e3..3b58c79ca 100644 --- a/test/helpers/gemini-session-runner.ts +++ b/test/helpers/gemini-session-runner.ts @@ -8,7 +8,8 @@ * Key differences from Codex session-runner: * - Uses `gemini -p` instead of `codex exec` * - Output is NDJSON with event types: init, message, tool_use, tool_result, result - * - Uses `--output-format stream-json --yolo` instead of `--json -s read-only` + * - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only` + * (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs) * - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd * - Message events are streamed with `delta: true` — must concatenate */ @@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: { } // Build gemini command - const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo']; + // --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json; + // without it gemini exits FatalUntrustedWorkspaceError before any model call. + const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; // Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted), // cwd for skill discovery. Hermetic scrub with gemini's auth surface diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 84b3059ab..b53d725d1 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -96,10 +96,17 @@ export function resultFromGeminiStream( /** * Gemini adapter — wraps the `gemini` CLI. * - * Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output - * format is NDJSON with `message`/`tool_use`/`result` events when `--output-format - * stream-json` is requested. This adapter uses a single-response form for simplicity - * in benchmarks; richer streaming lives in gemini-session-runner.ts. + * Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth. + * Personal OAuth free-tier is no longer supported by gemini CLI — use an + * AI Studio API key. Antigravity is a separate product/quota path. + * + * Headless flags always passed: + * --output-format stream-json — NDJSON events (message/tool_use/result) + * --yolo — auto-approve tools (non-interactive) + * --skip-trust — trust cwd for this session; required when + * workdir is a temp/untrusted folder (benchmarks + * use mkdtemp). Without it headless gemini exits + * before calling the model. */ export class GeminiAdapter implements ProviderAdapter { readonly name = 'gemini'; @@ -129,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); // Default to --yolo (non-interactive) and stream-json output so we can parse - // tokens + tool calls. Callers can override via extraArgs. - const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo']; + // tokens + tool calls. --skip-trust is required for headless/temp workdirs + // (gemini CLI otherwise exits: "not running in a trusted directory"). + // Callers can override via extraArgs. + const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust']; if (opts.model) args.push('--model', opts.model); if (opts.extraArgs) args.push(...opts.extraArgs);