Merge remote-tracking branch 'origin/main' into time-attack/wave-test-infra-isolated-review

This commit is contained in:
t
2026-07-14 16:51:31 -07:00
23 changed files with 681 additions and 96 deletions
+58
View File
@@ -0,0 +1,58 @@
import { describe, test, expect } from 'bun:test';
import { readFileSync } from 'fs';
import { join } from 'path';
const ROOT = join(import.meta.dir, '..');
// Security regression guard for the basic-ftp transitive dependency.
//
// basic-ftp reaches the tree only transitively:
// puppeteer-core > @puppeteer/browsers > proxy-agent > pac-proxy-agent > get-uri > basic-ftp
//
// Versions <= 5.3.0 carry four HIGH advisories, all fixed in 5.3.1:
// - GHSA-chqc-8p9q-pq6q CVE-2026-39983 FTP command injection via CRLF (fixed 5.2.1)
// - GHSA-6v7q-wjvx-w8wg incomplete CRLF protection, USER/PASS + MKD bypass (fixed 5.2.2)
// - GHSA-rpmf-866q-6p89 DoS via unbounded multiline control-response buffering
// - GHSA-rp42-5vxx-qpwr DoS via unbounded memory in Client.list()
//
// The fix is a bun `overrides` pin (not a phantom direct dependency): overriding
// forces EVERY basic-ftp in the tree to the safe version, including get-uri's
// nested copy. A direct-dependency bump leaves that nested copy behind — which is
// exactly the failure mode this test is here to catch. See the CVE-2026-39983
// upgrade fix for the full rationale.
const MIN_SAFE = '5.3.1';
function cmpSemver(a: string, b: string): number {
const pa = a.split('.').map(Number);
const pb = b.split('.').map(Number);
for (let i = 0; i < 3; i++) {
if ((pa[i] ?? 0) !== (pb[i] ?? 0)) return (pa[i] ?? 0) - (pb[i] ?? 0);
}
return 0;
}
describe('basic-ftp security pin (CVE-2026-39983 and siblings)', () => {
test('package.json overrides basic-ftp to a safe version', () => {
const pkg = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf-8'));
const pin = pkg.overrides?.['basic-ftp'];
expect(pin, 'package.json overrides.basic-ftp must exist').toBeDefined();
const pinned = String(pin).replace(/^[\^~]/, '');
expect(
cmpSemver(pinned, MIN_SAFE) >= 0,
`overrides.basic-ftp is "${pin}" but must be >= ${MIN_SAFE}`,
).toBe(true);
});
test('no basic-ftp entry in bun.lock resolves below the safe version', () => {
const lock = readFileSync(join(ROOT, 'bun.lock'), 'utf-8');
// Match every resolved basic-ftp specifier, including nested paths like
// "get-uri/basic-ftp" that a direct-dependency bump would leave vulnerable.
const versions = [...lock.matchAll(/basic-ftp@(\d+\.\d+\.\d+)/g)].map(m => m[1]);
expect(versions.length, 'expected at least one basic-ftp entry in bun.lock').toBeGreaterThan(0);
const vulnerable = versions.filter(v => cmpSemver(v, MIN_SAFE) < 0);
expect(
vulnerable,
`bun.lock still resolves vulnerable basic-ftp version(s): ${vulnerable.join(', ')}`,
).toEqual([]);
});
});
+21 -5
View File
@@ -6,13 +6,14 @@
* on PATH that emits canned exit codes + stderr matching the patterns the
* classifier looks for.
*
* Six status cases:
* Seven status cases:
* 1. no-cli — gbrain absent from PATH
* 2. missing-config — gbrain present, config.json absent (honors GBRAIN_HOME)
* 3. broken-config — gbrain present, config exists, stderr contains "config.json"
* 4. broken-db — gbrain present, config exists, stderr contains "Cannot connect to database"
* 5. timeout — probe exceeds GSTACK_GBRAIN_PROBE_TIMEOUT_MS with no recognized error (#1964)
* 6. ok — gbrain present, config exists, sources list returns valid JSON
* 6. engine-locked — PGLite CLI exits 124 because another process owns the DB (#2194)
* 7. ok — gbrain present, config exists, sources list returns valid JSON
*
* Plus cache behavior: hit, TTL expiry, invariant invalidation (HOME change,
* probe-timeout change), --no-cache bypass. Timeout tests keep runtime sane by
@@ -61,7 +62,7 @@ interface FakeEnv {
*/
function makeEnv(opts: {
withGbrain?: boolean;
gbrainBehavior?: "ok" | "broken-db" | "broken-config" | "throws" | "slow";
gbrainBehavior?: "ok" | "broken-db" | "broken-config" | "engine-locked" | "throws" | "slow";
withConfig?: boolean;
}): FakeEnv {
const tmp = mkdtempSync(join(tmpdir(), "gbrain-local-status-test-"));
@@ -102,7 +103,7 @@ function makeEnv(opts: {
}
function makeFakeGbrainScript(
behavior: "ok" | "broken-db" | "broken-config" | "throws" | "slow",
behavior: "ok" | "broken-db" | "broken-config" | "engine-locked" | "throws" | "slow",
): string {
// "slow": healthy engine on a cold pooler connection (#1964) — sleeps past
// the (test-lowered) probe timeout, then would answer fine.
@@ -125,10 +126,12 @@ exit 0
? 'echo "Cannot connect to database: . Fix: Check your connection URL in ~/.gbrain/config.json" >&2'
: behavior === "broken-config"
? 'echo "Error: malformed config.json at ~/.gbrain/config.json" >&2'
: behavior === "engine-locked"
? 'echo "gbrain sources: connect timed out (default 10000ms; pass --timeout=Ns to override)." >&2'
: behavior === "throws"
? 'echo "unexpected gbrain failure" >&2'
: "";
const exitCode = behavior === "ok" ? 0 : 1;
const exitCode = behavior === "ok" ? 0 : behavior === "engine-locked" ? 124 : 1;
return `#!/bin/sh
if [ "$1" = "--version" ]; then
echo "gbrain 0.33.1.0"
@@ -225,6 +228,19 @@ describe("lib/gbrain-local-status — status classification", () => {
expect(localEngineStatus({ noCache: true })).toBe("broken-config");
});
it("returns 'engine-locked' when PGLite exits 124 with its own connect timeout (#2194)", () => {
env = makeEnv({ withGbrain: true, gbrainBehavior: "engine-locked", withConfig: true });
restoreEnv = applyEnv(env);
expect(localEngineStatus({ noCache: true })).toBe("engine-locked");
});
it("classifies a non-PGLite connect timeout as unreachable DB, not malformed config", () => {
env = makeEnv({ withGbrain: true, gbrainBehavior: "engine-locked", withConfig: true });
restoreEnv = applyEnv(env);
writeFileSync(env.configPath, JSON.stringify({ engine: "postgres", database_url: "postgres://fake" }));
expect(localEngineStatus({ noCache: true })).toBe("broken-db");
});
it("returns 'ok' when sources list succeeds", () => {
env = makeEnv({ withGbrain: true, gbrainBehavior: "ok", withConfig: true });
restoreEnv = applyEnv(env);
+18 -1
View File
@@ -42,7 +42,7 @@ interface FakeEnv {
*/
function makeEnv(opts: {
withGbrain: boolean;
gbrainBehavior?: "ok" | "broken-db" | "broken-config" | "slow";
gbrainBehavior?: "ok" | "broken-db" | "broken-config" | "engine-locked" | "slow";
withConfig: boolean;
}): FakeEnv {
const tmp = mkdtempSync(join(tmpdir(), "gbrain-sync-skip-"));
@@ -75,6 +75,9 @@ function makeEnv(opts: {
: behavior === "ok"
? ` echo '{"sources":[]}'
exit 0`
: behavior === "engine-locked"
? ` echo "gbrain sources: connect timed out (default 10000ms; pass --timeout=Ns to override)." >&2
exit 124`
: ` ${
behavior === "broken-db"
? 'echo "Cannot connect to database: . Fix: Check your connection URL in ~/.gbrain/config.json" >&2'
@@ -207,6 +210,20 @@ describe("gstack-gbrain-sync — split-engine SKIP (plan D12)", () => {
}
});
it("SKIPs with actionable guidance when PGLite is held by gbrain serve (#2194)", () => {
const env = makeEnv({ withGbrain: true, gbrainBehavior: "engine-locked", withConfig: true });
try {
const r = runOrchestrator(env, ["--code-only"]);
const out = r.stdout + r.stderr;
expect(out).toContain("local engine engine-locked");
expect(out).toContain("gbrain serve");
expect(out).toContain("outside the live Claude session");
expect(out).not.toContain("config.json is malformed");
} finally {
env.cleanup();
}
});
it("SKIPs code stage when gbrain CLI is missing (no-cli)", () => {
const env = makeEnv({ withGbrain: false, withConfig: false });
try {
+176
View File
@@ -0,0 +1,176 @@
/**
* Project-slug consistency for gstack-repo-mode.
*
* These are end-to-end reproductions of #2212. Each case is a real local Git
* workspace which predates its public GitHub origin: gstack-slug caches the
* local directory name, then the origin is added. repo-mode must write its
* cache next to that canonical project state, not beside a newly derived
* owner-repo slug.
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'child_process';
import fs from 'fs';
import os from 'os';
import path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
const REPO_MODE_BIN = path.join(ROOT, 'bin', 'gstack-repo-mode');
const SLUG_BIN = path.join(ROOT, 'bin', 'gstack-slug');
type CommandResult = { stdout: string; stderr: string; status: number };
function run(command: string, args: string[], cwd: string, home: string): CommandResult {
const result = spawnSync(command, args, {
cwd,
encoding: 'utf8',
env: { ...process.env, HOME: home, GSTACK_HOME: path.join(home, '.gstack') },
});
return {
stdout: result.stdout ?? '',
stderr: result.stderr ?? '',
status: result.status ?? -1,
};
}
function git(args: string[], cwd: string, home: string) {
const result = run('git', args, cwd, home);
expect(result.status, `${result.stderr}\n${result.stdout}`).toBe(0);
}
function slugFrom(output: string): string {
const line = output.split('\n').find((entry) => entry.startsWith('SLUG='));
expect(line).toBeDefined();
return line!.slice('SLUG='.length);
}
const publicWorkspaces = [
{ name: 'react-local-history', origin: 'https://github.com/facebook/react.git' },
{ name: 'next-local-history', origin: 'https://github.com/vercel/next.js.git' },
{ name: 'kubernetes-local-history', origin: 'https://github.com/kubernetes/kubernetes.git' },
];
describe('gstack-repo-mode cached slug consistency (#2212)', () => {
for (const workspace of publicWorkspaces) {
test(`${workspace.origin} keeps its pre-origin project cache`, () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-home-'));
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-workspaces-'));
const project = path.join(root, workspace.name);
try {
fs.mkdirSync(project);
git(['init', '-b', 'main'], project, home);
git(['config', 'user.name', 'gstack regression test'], project, home);
git(['config', 'user.email', 'test@gstack.dev'], project, home);
// This workspace existed and used gstack before it adopted GitHub.
const initialSlug = slugFrom(run(SLUG_BIN, [], project, home).stdout);
expect(initialSlug).toBe(workspace.name);
// Five commits make repo-mode perform its real classification path.
for (let commit = 1; commit <= 5; commit += 1) {
fs.writeFileSync(path.join(project, `commit-${commit}.txt`), String(commit));
git(['add', '.'], project, home);
git(['commit', '-m', `commit ${commit}`], project, home);
}
git(['remote', 'add', 'origin', workspace.origin], project, home);
const mode = run(REPO_MODE_BIN, [], project, home);
expect(mode.status, mode.stderr).toBe(0);
expect(mode.stdout.trim()).toBe('REPO_MODE=solo');
// Canonical slug stays the pre-origin basename; remote-derived twin must not appear.
const projectsDir = path.join(home, '.gstack', 'projects');
const canonicalCache = path.join(projectsDir, initialSlug, 'repo-mode.json');
expect(fs.existsSync(canonicalCache)).toBe(true);
expect(fs.readdirSync(projectsDir)).toEqual([initialSlug]);
// Slug remains sanitized even when the remote URL would otherwise diverge.
expect(initialSlug).toMatch(/^[a-zA-Z0-9._-]+$/);
expect(slugFrom(run(SLUG_BIN, [], project, home).stdout)).toBe(initialSlug);
} finally {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(root, { recursive: true, force: true });
}
});
}
test('still treats a workspace without origin as unknown', () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-home-'));
const project = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-local-only-'));
try {
git(['init', '-b', 'main'], project, home);
const mode = run(REPO_MODE_BIN, [], project, home);
expect(mode.status, mode.stderr).toBe(0);
expect(mode.stdout.trim()).toBe('REPO_MODE=unknown');
expect(fs.existsSync(path.join(home, '.gstack', 'projects'))).toBe(false);
} finally {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(project, { recursive: true, force: true });
}
});
test('subdir invocation still writes under the git-root cached slug', () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-home-'));
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-subdir-'));
const project = path.join(root, 'nested-history');
const nested = path.join(project, 'packages', 'app');
try {
fs.mkdirSync(nested, { recursive: true });
git(['init', '-b', 'main'], project, home);
git(['config', 'user.name', 'gstack regression test'], project, home);
git(['config', 'user.email', 'test@gstack.dev'], project, home);
const initialSlug = slugFrom(run(SLUG_BIN, [], project, home).stdout);
expect(initialSlug).toBe('nested-history');
for (let commit = 1; commit <= 5; commit += 1) {
fs.writeFileSync(path.join(project, `commit-${commit}.txt`), String(commit));
git(['add', '.'], project, home);
git(['commit', '-m', `commit ${commit}`], project, home);
}
git(['remote', 'add', 'origin', 'https://github.com/facebook/react.git'], project, home);
const mode = run(REPO_MODE_BIN, [], nested, home);
expect(mode.status, mode.stderr).toBe(0);
expect(mode.stdout.trim()).toBe('REPO_MODE=solo');
const projectsDir = path.join(home, '.gstack', 'projects');
expect(fs.existsSync(path.join(projectsDir, initialSlug, 'repo-mode.json'))).toBe(true);
expect(fs.readdirSync(projectsDir)).toEqual([initialSlug]);
} finally {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(root, { recursive: true, force: true });
}
});
test('sanitizes unusual origin URLs via gstack-slug before mkdir', () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-home-'));
const project = fs.mkdtempSync(path.join(os.tmpdir(), 'grepo-weird-url-'));
try {
git(['init', '-b', 'main'], project, home);
git(['config', 'user.name', 'gstack regression test'], project, home);
git(['config', 'user.email', 'test@gstack.dev'], project, home);
for (let commit = 1; commit <= 5; commit += 1) {
fs.writeFileSync(path.join(project, `commit-${commit}.txt`), String(commit));
git(['add', '.'], project, home);
git(['commit', '-m', `commit ${commit}`], project, home);
}
// Free-form remote URL with characters outside [a-zA-Z0-9._-]
git(['remote', 'add', 'origin', 'https://example.com/org/my repo (v2).git'], project, home);
const mode = run(REPO_MODE_BIN, [], project, home);
expect(mode.status, mode.stderr).toBe(0);
expect(mode.stdout.trim()).toBe('REPO_MODE=solo');
const projects = fs.readdirSync(path.join(home, '.gstack', 'projects'));
expect(projects).toHaveLength(1);
expect(projects[0]).toMatch(/^[a-zA-Z0-9._-]+$/);
expect(projects[0]).not.toMatch(/[ ()]/);
} finally {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(project, { recursive: true, force: true });
}
});
});
+5 -2
View File
@@ -8,7 +8,8 @@
* Key differences from Codex session-runner:
* - Uses `gemini -p` instead of `codex exec`
* - Output is NDJSON with event types: init, message, tool_use, tool_result, result
* - Uses `--output-format stream-json --yolo` instead of `--json -s read-only`
* - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only`
* (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs)
* - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
* - Message events are streamed with `delta: true` — must concatenate
*/
@@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: {
}
// Build gemini command
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo'];
// --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json;
// without it gemini exits FatalUntrustedWorkspaceError before any model call.
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
// Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
// cwd for skill discovery. Hermetic scrub with gemini's auth surface
+138
View File
@@ -0,0 +1,138 @@
import { describe, test, expect } from 'bun:test';
import { parseGeminiStreamJson, resultFromGeminiStream } from './gemini';
// Current CLI shape (gemini ≥ ~0.11 / stream-json): content + role, tokens under result.stats
// Documented at https://geminicli.com/docs/cli/headless/ and gemini-cli PR #10883.
const CURRENT_CLI_FIXTURE = [
'{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}',
'{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}',
'{"type":"message","role":"assistant","content":"PONG","delta":true}',
'{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}',
].join('\n');
// Legacy shape: text field, tokens under result.usage
const LEGACY_FIXTURE = [
'{"type":"message","text":"hello"}',
'{"type":"tool_use","name":"run_shell_command"}',
'{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}',
].join('\n');
describe('parseGeminiStreamJson', () => {
test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => {
const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE);
expect(parsed.output).toBe('PONG');
expect(parsed.tokens.input).toBe(24946);
expect(parsed.tokens.output).toBe(32);
expect(parsed.modelUsed).toBe('gemini-2.5-pro');
expect(parsed.toolCalls).toBe(0);
});
test('current CLI: user role content must not concat into output', () => {
const raw = [
'{"type":"message","role":"user","content":"PROMPT ECHO"}',
'{"type":"message","role":"assistant","content":"ok","delta":true}',
].join('\n');
const parsed = parseGeminiStreamJson(raw);
expect(parsed.output).toBe('ok');
expect(parsed.output).not.toContain('PROMPT ECHO');
});
test('legacy: still accepts text + usage.input_token_count', () => {
const parsed = parseGeminiStreamJson(LEGACY_FIXTURE);
expect(parsed.output).toBe('hello');
expect(parsed.tokens.input).toBe(100);
expect(parsed.tokens.output).toBe(5);
expect(parsed.toolCalls).toBe(1);
expect(parsed.modelUsed).toBe('gemini-2.5-flash');
});
test('legacy text with role:user is ignored', () => {
const raw = [
'{"type":"message","role":"user","text":"echo"}',
'{"type":"message","role":"assistant","text":"kept"}',
].join('\n');
const parsed = parseGeminiStreamJson(raw);
expect(parsed.output).toBe('kept');
});
test('concatenates multiple assistant content deltas', () => {
const raw = [
'{"type":"message","role":"assistant","content":"A","delta":true}',
'{"type":"message","role":"assistant","content":"B","delta":true}',
].join('\n');
expect(parseGeminiStreamJson(raw).output).toBe('AB');
});
test('skips malformed lines without throwing', () => {
const raw = [
'{"type":"init","model":"m1"}',
'not json',
'{"type":"message","role":"assistant","content":"x","delta":true}',
'{incomplete',
'{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}',
].join('\n');
const parsed = parseGeminiStreamJson(raw);
expect(parsed.output).toBe('x');
expect(parsed.tokens).toEqual({ input: 1, output: 2 });
expect(parsed.modelUsed).toBe('m1');
});
test('empty / whitespace-only input yields empty parse (no throw)', () => {
const parsed = parseGeminiStreamJson('');
expect(parsed.output).toBe('');
expect(parsed.tokens).toEqual({ input: 0, output: 0 });
expect(parsed.toolCalls).toBe(0);
expect(parsed.modelUsed).toBeUndefined();
});
test('result.model overrides init.model when both present', () => {
const raw = [
'{"type":"init","model":"from-init"}',
'{"type":"message","role":"assistant","content":"hi"}',
'{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}',
].join('\n');
expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result');
});
});
describe('resultFromGeminiStream (adapter post-CLI e2e)', () => {
test('current CLI fixture → success row with PONG + tokens (not $0)', () => {
const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE, { durationMs: 42 });
expect(result.error).toBeUndefined();
expect(result.output).toBe('PONG');
expect(result.tokens.input).toBe(24946);
expect(result.tokens.output).toBe(32);
expect(result.modelUsed).toBe('gemini-2.5-pro');
expect(result.durationMs).toBe(42);
// Cost signal: non-zero tokens means estimateCost will not be $0.
expect(result.tokens.input + result.tokens.output).toBeGreaterThan(0);
});
test('legacy text-only success still works', () => {
const result = resultFromGeminiStream(LEGACY_FIXTURE, { durationMs: 10 });
expect(result.error).toBeUndefined();
expect(result.output).toBe('hello');
expect(result.toolCalls).toBe(1);
});
test('empty exit-0 stream → error row, never silent $0 success (#2159)', () => {
const emptySuccess = [
'{"type":"init","model":"gemini-2.5-pro"}',
'{"type":"message","role":"user","content":"hi"}',
'{"type":"result","status":"success","stats":{"input_tokens":0,"output_tokens":0}}',
].join('\n');
const result = resultFromGeminiStream(emptySuccess, { durationMs: 5 });
expect(result.error).toBeDefined();
expect(result.error!.code).toBe('unknown');
expect(result.error!.reason).toContain('empty output');
expect(result.output).toBe('');
expect(result.modelUsed).toBe('gemini-2.5-pro');
});
test('pre-fix bug shape (text field missing, only content) would have been empty — now succeeds', () => {
// This is the exact failure mode from #2159: content present, no text.
const result = resultFromGeminiStream(CURRENT_CLI_FIXTURE);
expect(result.error).toBeUndefined();
expect(result.output).toBe('PONG');
});
});
+119 -52
View File
@@ -5,13 +5,108 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
export type GeminiStreamParse = {
output: string;
tokens: { input: number; output: number };
toolCalls: number;
modelUsed?: string;
};
/**
* Parse gemini NDJSON stream events (exported for unit tests).
*
* Current CLI (`--output-format stream-json`) emits:
* init model
* message { role, content, delta? } concat assistant content
* tool_use increment toolCalls
* result { stats: { input_tokens, output_tokens } } tokens
*
* Legacy shape (still accepted):
* message { text } concat text
* result { usage: { input_token_count, output_token_count } } tokens
*/
export function parseGeminiStreamJson(raw: string): GeminiStreamParse {
let output = '';
let input = 0;
let out = 0;
let toolCalls = 0;
let modelUsed: string | undefined;
for (const line of raw.split('\n')) {
const s = line.trim();
if (!s) continue;
try {
const obj = JSON.parse(s);
if (obj.type === 'init') {
if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
} else if (obj.type === 'message') {
// Current CLI: content + role. Role guard is required — the CLI echoes
// the user prompt as role:'user', which must not land in output.
if (obj.role === 'assistant' && typeof obj.content === 'string') {
output += obj.content;
} else if (typeof obj.text === 'string' && obj.role !== 'user') {
// Legacy text field (no role, or assistant).
output += obj.text;
}
} else if (obj.type === 'tool_use') {
toolCalls += 1;
} else if (obj.type === 'result') {
const u = obj.usage ?? obj.stats ?? {};
input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0;
out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0;
if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
}
} catch {
// skip malformed lines
}
}
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
}
/**
* Map a raw stream-json dump to a RunResult, including the empty-success
* hardening from #2159. Exported so adapter e2e can exercise the full
* post-CLI path without a live gemini binary.
*/
export function resultFromGeminiStream(
raw: string,
opts: { model?: string; durationMs?: number } = {},
): RunResult {
const parsed = parseGeminiStreamJson(raw);
const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro';
const durationMs = opts.durationMs ?? 0;
if (!parsed.output.trim()) {
return {
output: '',
tokens: { input: 0, output: 0 },
durationMs,
toolCalls: 0,
modelUsed,
error: { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' },
};
}
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs,
toolCalls: parsed.toolCalls,
modelUsed,
};
}
/**
* Gemini adapter wraps the `gemini` CLI.
*
* Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
* stream-json` is requested. This adapter uses a single-response form for simplicity
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
* Personal OAuth free-tier is no longer supported by gemini CLI use an
* AI Studio API key. Antigravity is a separate product/quota path.
*
* Headless flags always passed:
* --output-format stream-json NDJSON events (message/tool_use/result)
* --yolo auto-approve tools (non-interactive)
* --skip-trust trust cwd for this session; required when
* workdir is a temp/untrusted folder (benchmarks
* use mkdtemp). Without it headless gemini exits
* before calling the model.
*/
export class GeminiAdapter implements ProviderAdapter {
readonly name = 'gemini';
@@ -26,9 +121,14 @@ export class GeminiAdapter implements ProviderAdapter {
const newCfgDir = path.join(os.homedir(), '.gemini');
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
const hasKey = !!process.env.GOOGLE_API_KEY;
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
if (!hasCfg && !hasKey) {
return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
return {
ok: false,
reason:
'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
};
}
return { ok: true };
}
@@ -36,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter {
async run(opts: RunOpts): Promise<RunResult> {
const start = Date.now();
// Default to --yolo (non-interactive) and stream-json output so we can parse
// tokens + tool calls. Callers can override via extraArgs.
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
// tokens + tool calls. --skip-trust is required for headless/temp workdirs
// (gemini CLI otherwise exits: "not running in a trusted directory").
// Callers can override via extraArgs.
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
if (opts.model) args.push('--model', opts.model);
if (opts.extraArgs) args.push(...opts.extraArgs);
@@ -47,15 +149,15 @@ export class GeminiAdapter implements ProviderAdapter {
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
env: {
...process.env,
// Prefer GEMINI_API_KEY when only that is set (CLI reads both).
...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY
? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY }
: {}),
},
});
const parsed = this.parseStreamJson(out);
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
};
return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start });
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -63,7 +165,7 @@ export class GeminiAdapter implements ProviderAdapter {
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
}
if (/unauthorized|auth|login|api key/i.test(stderr)) {
if (/unauthorized|auth|login|api key|ineligibletier|no longer supported/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
}
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
@@ -77,41 +179,6 @@ export class GeminiAdapter implements ProviderAdapter {
return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
}
/**
* Parse gemini NDJSON stream events:
* init session id (discarded here)
* message { delta: true, text } concat to output
* tool_use { name } increment toolCalls
* result { usage: { input_token_count, output_token_count } } tokens
*/
private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
let output = '';
let input = 0;
let out = 0;
let toolCalls = 0;
let modelUsed: string | undefined;
for (const line of raw.split('\n')) {
const s = line.trim();
if (!s) continue;
try {
const obj = JSON.parse(s);
if (obj.type === 'message' && typeof obj.text === 'string') {
output += obj.text;
} else if (obj.type === 'tool_use') {
toolCalls += 1;
} else if (obj.type === 'result') {
const u = obj.usage ?? {};
input += u.input_token_count ?? u.prompt_tokens ?? 0;
out += u.output_token_count ?? u.completion_tokens ?? 0;
if (obj.model) modelUsed = obj.model;
}
} catch {
// skip malformed lines
}
}
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
}
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
return {
output: '',
+7 -11
View File
@@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
if (result.error) {
throw new Error(`gemini errored: ${result.error.code}${result.error.reason}`);
}
// Gemini CLI occasionally returns empty output even on successful runs
// (model returned content the CLI parser missed, intermittent stream issues).
// We assert the adapter ran end-to-end without erroring and reports a non-
// empty token count instead of grepping the literal "ok" — that string
// assertion was too brittle for a smoke that's really about "did the
// adapter wire up and the run terminate successfully?"
expect(typeof result.output).toBe('string');
// Gemini CLI sometimes returns 0 tokens in the result event (older responses);
// assert non-negative instead of strictly positive.
expect(result.tokens.input).toBeGreaterThanOrEqual(0);
expect(result.tokens.output).toBeGreaterThanOrEqual(0);
// Adapter must never report empty-success (#2159). After content/stats
// parsing, a healthy run has non-empty assistant text + token counts.
expect(result.output.trim().length).toBeGreaterThan(0);
expect(result.output.toLowerCase()).toContain('ok');
expect(result.tokens.input).toBeGreaterThan(0);
expect(result.tokens.output).toBeGreaterThan(0);
expect(result.durationMs).toBeGreaterThan(0);
expect(typeof result.modelUsed).toBe('string');
expect(result.modelUsed.length).toBeGreaterThan(0);
}, 150_000);
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
+10 -4
View File
@@ -8,7 +8,7 @@
* 3. sha8($(git config user.email))
* 4. anonymous-<sha8(hostname)>
*
* Result is persisted under user_slug_at_<endpoint-hash> for stability.
* Result is persisted under user_slug_at_<endpoint-id> for stability.
* Test isolation via GSTACK_HOME and HOME env overrides.
*
* Gate-tier, free, ~50ms.
@@ -87,12 +87,12 @@ describe('resolve-user-slug fallback chain', () => {
expect(slug).toMatch(/^(email-|anonymous-)[a-f0-9]+$|^[a-zA-Z0-9-]+$/);
});
test('persists resolution to user_slug_at_<hash> on first call', () => {
test('persists resolution to user_slug_at_<endpoint-id> on first call', () => {
runConfig(['resolve-user-slug'], { GSTACK_HOME: TMP_HOME, USER: 'persisttest' });
const configFile = join(TMP_HOME, 'config.yaml');
expect(existsSync(configFile)).toBe(true);
const content = readFileSync(configFile, 'utf-8');
expect(content).toMatch(/^user_slug_at_[a-f0-9]+:\s+persisttest/m);
expect(content).toMatch(/^user_slug_at_(local|[a-f0-9]{8}|[a-f0-9]{16}):\s+persisttest/m);
});
test('subsequent calls return same slug (stable across sessions)', () => {
@@ -104,7 +104,7 @@ describe('resolve-user-slug fallback chain', () => {
});
});
describe('brain_trust_policy@<hash> namespace', () => {
describe('brain_trust_policy@<endpoint-id> namespace', () => {
test('default value is "unset"', () => {
const result = runConfig(['get', 'brain_trust_policy@deadbeef'], { GSTACK_HOME: TMP_HOME });
expect(result.status).toBe(0);
@@ -158,4 +158,10 @@ describe('key validation', () => {
const result = runConfig(['get', 'brain_trust_policy@abc123ff'], { GSTACK_HOME: TMP_HOME });
expect(result.status).toBe(0);
});
test('accepts @local suffix on key', () => {
const result = runConfig(['get', 'brain_trust_policy@local'], { GSTACK_HOME: TMP_HOME });
expect(result.status).toBe(0);
expect(result.stdout).toBe('unset');
});
});