fix(evals): absorb codex/gemini CLI drift; external-service tests go periodic-tier

- codex exec gains --skip-git-repo-check: newer CLIs refuse exec in an
  untrusted non-git dir (our temp skill dirs) — empirically verified.
- gemini: --skip-trust was removed in gemini-cli 0.34 (argv parse error);
  dropped from the session runner and the benchmark adapter. A present-
  but-unusable CLI (deprecated individual code-assist auth path) now
  classifies as SKIP, not a false adapter failure; the benchmark live
  smoke skips on auth/rate_limit error codes (environmental) while still
  failing on timeout/unknown (the drift classes it exists to catch).
- codex-e2e, gemini-e2e, and benchmark-providers gain the canonical
  whole-file EVALS_TIER === 'periodic' guard per CLAUDE.md tiering rule 3
  (external service -> periodic) — the sharded gate runner now excludes
  all three (gate: 45 -> 42 shards).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-13 10:34:25 -07:00
co-authored by Claude Fable 5
parent 990d54a9e4
commit bd11416d80
6 changed files with 100 additions and 21 deletions
+32 -6
View File
@@ -8,8 +8,8 @@
* Key differences from Codex session-runner:
* - Uses `gemini -p` instead of `codex exec`
* - Output is NDJSON with event types: init, message, tool_use, tool_result, result
* - Uses `--output-format stream-json --yolo --skip-trust` instead of `--json -s read-only`
* (`--skip-trust` required for headless/untrusted cwds; see gemini trusted-folders docs)
* - Uses `--output-format stream-json --yolo` instead of `--json -s read-only`
* (`--skip-trust` was removed in gemini-cli 0.34; folder trust is settings-driven now)
* - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
* - Message events are streamed with `delta: true` — must concatenate
*/
@@ -121,10 +121,11 @@ export async function runGeminiSkill(opts: {
};
}
// Build gemini command
// --skip-trust: headless/CI and temp cwds aren't in ~/.gemini/trustedFolders.json;
// without it gemini exits FatalUntrustedWorkspaceError before any model call.
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo', '--skip-trust'];
// Build gemini command.
// --skip-trust was REMOVED in gemini-cli 0.34 ("Unknown arguments:
// skip-trust"); folder trust moved to settings and no longer needs a flag
// for headless runs. --yolo still auto-approves tool actions.
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo'];
// Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
// cwd for skill discovery. Hermetic scrub with gemini's auth surface
@@ -198,6 +199,31 @@ export async function runGeminiSkill(opts: {
process.stderr.write(` [gemini stderr] ${stderr.trim().slice(0, 200)}\n`);
}
// Environment-unusable classification: these are Google-side conditions no
// test assertion can act on — the deprecated individual code-assist auth
// path ("migrate to the Antigravity suite") and argv drift on older/newer
// CLIs. Return the same SKIP shape as binary-not-found so callers report
// SKIPPED instead of a false FAIL.
const unusableMarkers = [
'no longer supported for Gemini Code Assist',
'antigravity',
'Unknown arguments: skip-trust',
];
if (exitCode !== 0 && parsed.tokens === 0) {
const marker = unusableMarkers.find((m) => stderr.toLowerCase().includes(m.toLowerCase()));
if (marker) {
return {
output: `SKIP: gemini CLI unusable (${marker})`,
toolCalls: [],
tokens: 0,
exitCode: -1,
durationMs,
sessionId: null,
rawLines: collectedLines,
};
}
}
return {
output: parsed.output,
toolCalls: parsed.toolCalls,