mirror of
https://github.com/garrytan/gstack.git
synced 2026-07-18 13:37:23 +02:00
fix(benchmark): skip-trust for headless gemini + drop release metadata
Pass --skip-trust so temp/untrusted workdirs work in headless benchmarks (mirrors Codex --skip-git-repo-check). Revert VERSION, CHANGELOG, and package.json bumps — leave release bookkeeping to maintainers / /ship, matching community PR practice. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,23 +1,5 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
## [1.60.2.0] - 2026-07-10
|
|
||||||
|
|
||||||
## **Gemini model benchmarks no longer rank a silent empty run as a $0 win.**
|
|
||||||
|
|
||||||
`gstack-model-benchmark --models gemini` was parsing an older stream-json shape (`message.text`, `result.usage`). Current gemini CLI emits `message.content` (with `role`) and `result.stats`, so the adapter returned empty output, 0 tokens, and no error — a fake success that quietly mis-ranked gemini in comparison tables.
|
|
||||||
|
|
||||||
### What this means for you
|
|
||||||
|
|
||||||
Gemini benchmark rows now show real assistant text and token counts again. If a run somehow still exits 0 with empty output, it surfaces as an error row instead of a $0 success.
|
|
||||||
|
|
||||||
### Itemized changes
|
|
||||||
|
|
||||||
#### Fixed
|
|
||||||
|
|
||||||
- `test/helpers/providers/gemini.ts`: `parseGeminiStreamJson` accepts current CLI `content` + `role:'assistant'` (ignores user-prompt echoes), reads tokens from `result.stats` with legacy `usage` fallbacks, and takes `model` from `init`. Empty exit-0 output is reported as `error.code=unknown` instead of a silent success (#2159).
|
|
||||||
- `test/helpers/providers/gemini.test.ts`: unit coverage for current + legacy stream shapes, user-echo guard, and empty parse.
|
|
||||||
- `test/skill-e2e-benchmark-providers.test.ts`: live gemini smoke asserts non-empty output containing the probe string and positive token counts.
|
|
||||||
|
|
||||||
## [1.60.1.0] - 2026-07-09
|
## [1.60.1.0] - 2026-07-09
|
||||||
|
|
||||||
## **The /autoplan dual-voice eval is back on the board, catching real regressions.**
|
## **The /autoplan dual-voice eval is back on the board, catching real regressions.**
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "gstack",
|
"name": "gstack",
|
||||||
"version": "1.60.2.0",
|
"version": "1.60.1.0",
|
||||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
|
|||||||
@@ -8,7 +8,8 @@
|
|||||||
* Key differences from Codex session-runner:
|
* Key differences from Codex session-runner:
|
||||||
* - Uses `gemini -p` instead of `codex exec`
|
* - Uses `gemini -p` instead of `codex exec`
|
||||||
* - Output is NDJSON with event types: init, message, tool_use, tool_result, result
|
* - Output is NDJSON with event types: init, message, tool_use, tool_result, result
|
||||||
* - Uses `--output-format stream-json --yolo` instead of `--json -s read-only`
|
* - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only`
|
||||||
|
* (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs)
|
||||||
* - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
|
* - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
|
||||||
* - Message events are streamed with `delta: true` — must concatenate
|
* - Message events are streamed with `delta: true` — must concatenate
|
||||||
*/
|
*/
|
||||||
@@ -121,7 +122,9 @@ export async function runGeminiSkill(opts: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build gemini command
|
// Build gemini command
|
||||||
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo'];
|
// --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json;
|
||||||
|
// without it gemini exits FatalUntrustedWorkspaceError before any model call.
|
||||||
|
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
|
||||||
|
|
||||||
// Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
|
// Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
|
||||||
// cwd for skill discovery. Hermetic scrub with gemini's auth surface
|
// cwd for skill discovery. Hermetic scrub with gemini's auth surface
|
||||||
|
|||||||
@@ -96,10 +96,17 @@ export function resultFromGeminiStream(
|
|||||||
/**
|
/**
|
||||||
* Gemini adapter — wraps the `gemini` CLI.
|
* Gemini adapter — wraps the `gemini` CLI.
|
||||||
*
|
*
|
||||||
* Gemini CLI auth comes from either ~/.config/gemini/ or GOOGLE_API_KEY. Output
|
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
|
||||||
* format is NDJSON with `message`/`tool_use`/`result` events when `--output-format
|
* Personal OAuth free-tier is no longer supported by gemini CLI — use an
|
||||||
* stream-json` is requested. This adapter uses a single-response form for simplicity
|
* AI Studio API key. Antigravity is a separate product/quota path.
|
||||||
* in benchmarks; richer streaming lives in gemini-session-runner.ts.
|
*
|
||||||
|
* Headless flags always passed:
|
||||||
|
* --output-format stream-json — NDJSON events (message/tool_use/result)
|
||||||
|
* --yolo — auto-approve tools (non-interactive)
|
||||||
|
* --skip-trust — trust cwd for this session; required when
|
||||||
|
* workdir is a temp/untrusted folder (benchmarks
|
||||||
|
* use mkdtemp). Without it headless gemini exits
|
||||||
|
* before calling the model.
|
||||||
*/
|
*/
|
||||||
export class GeminiAdapter implements ProviderAdapter {
|
export class GeminiAdapter implements ProviderAdapter {
|
||||||
readonly name = 'gemini';
|
readonly name = 'gemini';
|
||||||
@@ -129,8 +136,10 @@ export class GeminiAdapter implements ProviderAdapter {
|
|||||||
async run(opts: RunOpts): Promise<RunResult> {
|
async run(opts: RunOpts): Promise<RunResult> {
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
// Default to --yolo (non-interactive) and stream-json output so we can parse
|
// Default to --yolo (non-interactive) and stream-json output so we can parse
|
||||||
// tokens + tool calls. Callers can override via extraArgs.
|
// tokens + tool calls. --skip-trust is required for headless/temp workdirs
|
||||||
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
|
// (gemini CLI otherwise exits: "not running in a trusted directory").
|
||||||
|
// Callers can override via extraArgs.
|
||||||
|
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
|
||||||
if (opts.model) args.push('--model', opts.model);
|
if (opts.model) args.push('--model', opts.model);
|
||||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user