feat: hermetic Codex runner hardening + Sol scope-termination E2E

The Codex E2E runner copies auth.json only (operator plugins, MCP servers,
rules, and skills no longer leak into hermetic evals), pins CODEX_HOME to
the temp dir, and supports per-run model, TOML overrides, and
--ignore-user-config. New periodic E2E installs the FULL generated
investigate skill on gpt-5.6-sol against a planted one-line bug with decoy
TODOs: the fix must land inside the boundary (untracked files counted via
git status --porcelain), decoys stay byte-identical, the regression oracle
survives unweakened, nothing gets committed, all within 30 tool calls.
The shared .agents tree is snapshotted and restored exactly in beforeAll;
fixture commits disable gpg signing. Wired into the periodic CI matrix,
paid-shard globs, eval scripts, touchfiles/E2E_TIERS
(codex-sol-scope-termination), and diff-based selection. Real-file
periodic-tier classification pins both codex E2Es out of the gate tier.
Free-tier test proves an explicit --model overrides the host default
through the real generation CLI.

test/helpers/codex-session-runner.ts

@@ -159,6 +159,9 @@ export async function runCodexSkill(opts: {
   skillName?: string;       // Skill name for installation (default: dirname)
   sandbox?: string;         // Sandbox mode (default: 'read-only')
   sections?: string[];      // Install only these `## <section>` blocks (extract, don't copy)
+  model?: string;           // Exact Codex model ID (passed with --model)
+  configOverrides?: string[]; // TOML key=value overrides (passed with -c)
+  ignoreUserConfig?: boolean; // Add --ignore-user-config; auth still comes from CODEX_HOME
 }): Promise<CodexResult> {
   const {
     skillDir,
@@ -168,6 +171,9 @@ export async function runCodexSkill(opts: {
     skillName,
     sandbox = 'read-only',
     sections,
+    model,
+    configOverrides = [],
+    ignoreUserConfig = false,
   } = opts;
 
   const startTime = Date.now();
@@ -196,20 +202,16 @@ export async function runCodexSkill(opts: {
   try {
     installSkillToTempHome(skillDir, name, tempHome, sections);
 
-    // Symlink real Codex auth config so codex can authenticate from temp HOME.
-    // Codex stores auth in ~/.codex/ — we need the config but not the skills
-    // (we install our own test skills above).
-    const realCodexConfig = path.join(realHome, '.codex');
+    // Copy authentication only. Copying the whole operator ~/.codex tree leaks
+    // plugins, MCP servers, rules, memories, and skills into a supposedly
+    // hermetic E2E; required private MCPs can then fail before the model starts.
+    const realCodexConfig = process.env.CODEX_HOME || path.join(realHome, '.codex');
     const tempCodexDir = path.join(tempHome, '.codex');
     if (fs.existsSync(realCodexConfig)) {
-      // Copy auth-related files from real ~/.codex/ into temp ~/.codex/
-      // (skills/ is already set up by installSkillToTempHome)
-      const entries = fs.readdirSync(realCodexConfig);
-      for (const entry of entries) {
-        if (entry === 'skills') continue; // don't clobber our test skills
+      for (const entry of ['auth.json']) {
         const src = path.join(realCodexConfig, entry);
         const dst = path.join(tempCodexDir, entry);
-        if (!fs.existsSync(dst)) {
+        if (fs.existsSync(src) && !fs.existsSync(dst)) {
           fs.cpSync(src, dst, { recursive: true });
         }
       }
@@ -220,7 +222,11 @@ export async function runCodexSkill(opts: {
     // non-git directory ("Not inside a trusted directory and
     // --skip-git-repo-check was not specified") — our temp skill dirs are
     // exactly that. Empirically verified against codex on this machine.
-    const args = ['exec', prompt, '--json', '-s', sandbox, '--skip-git-repo-check'];
+    const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check'];
+    if (ignoreUserConfig) args.push('--ignore-user-config');
+    if (model) args.push('--model', model);
+    for (const override of configOverrides) args.push('-c', override);
+    args.push(prompt);
 
     // Spawn codex with temp HOME so it discovers our installed skill.
     // Hermetic scrub (test/helpers/hermetic-env.ts) with codex's auth surface
@@ -231,7 +237,7 @@ export async function runCodexSkill(opts: {
       stdout: 'pipe',
       stderr: 'pipe',
       env: hermeticChildEnv(
-        { HOME: tempHome },
+        { HOME: tempHome, CODEX_HOME: tempCodexDir },
         { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] },
       ),
     });

test/helpers/eval-store.ts

@@ -65,6 +65,7 @@ export interface EvalTestEntry {
   prompt?: string;
   output?: string;
   turns_used?: number;
+  tokens_used?: number;
   browse_errors?: string[];
 
   // LLM judge

test/helpers/paid-test-set.ts

@@ -15,6 +15,7 @@ export const PAID_TEST_GLOBS = [
   'test/skill-e2e-*.test.ts',
   'test/skill-routing-e2e.test.ts',
   'test/codex-e2e.test.ts',
+  'test/codex-e2e-sol-scope.test.ts',
   'test/gemini-e2e.test.ts',
 ] as const;
 

test/helpers/touchfiles-data.ts

@@ -269,6 +269,9 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
   'codex-discover-skill':  ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
   'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
 
+  // GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill)
+  'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'],
+
   // Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
   'gemini-smoke':  ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
 
@@ -666,6 +669,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
   // Multi-AI — periodic (require external CLIs)
   'codex-discover-skill': 'periodic',
   'codex-review-findings': 'periodic',
+  'codex-sol-scope-termination': 'periodic',
   'gemini-smoke': 'periodic',
 
   // Design — gate for cheap functional, periodic for Opus/quality