feat: hermetic Codex runner hardening + Sol scope-termination E2E

The Codex E2E runner copies auth.json only (operator plugins, MCP servers,
rules, and skills no longer leak into hermetic evals), pins CODEX_HOME to
the temp dir, and supports per-run model, TOML overrides, and
--ignore-user-config. New periodic E2E installs the FULL generated
investigate skill on gpt-5.6-sol against a planted one-line bug with decoy
TODOs: the fix must land inside the boundary (untracked files counted via
git status --porcelain), decoys stay byte-identical, the regression oracle
survives unweakened, nothing gets committed, all within 30 tool calls.
The shared .agents tree is snapshotted and restored exactly in beforeAll;
fixture commits disable gpg signing. Wired into the periodic CI matrix,
paid-shard globs, eval scripts, touchfiles/E2E_TIERS
(codex-sol-scope-termination), and diff-based selection. Real-file
periodic-tier classification pins both codex E2Es out of the gate tier.
Free-tier test proves an explicit --model overrides the host default
through the real generation CLI.
This commit is contained in:
Garry Tan
2026-08-18 13:11:15 -07:00
parent 39a60588ed
commit 8229fdc489
10 changed files with 346 additions and 22 deletions
+4
View File
@@ -269,6 +269,9 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
// GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill)
'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'],
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
@@ -666,6 +669,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
// Multi-AI — periodic (require external CLIs)
'codex-discover-skill': 'periodic',
'codex-review-findings': 'periodic',
'codex-sol-scope-termination': 'periodic',
'gemini-smoke': 'periodic',
// Design — gate for cheap functional, periodic for Opus/quality