feat: hermetic Codex runner hardening + Sol scope-termination E2E

The Codex E2E runner copies auth.json only (operator plugins, MCP servers,
rules, and skills no longer leak into hermetic evals), pins CODEX_HOME to
the temp dir, and supports per-run model, TOML overrides, and
--ignore-user-config. New periodic E2E installs the FULL generated
investigate skill on gpt-5.6-sol against a planted one-line bug with decoy
TODOs: the fix must land inside the boundary (untracked files counted via
git status --porcelain), decoys stay byte-identical, the regression oracle
survives unweakened, nothing gets committed, all within 30 tool calls.
The shared .agents tree is snapshotted and restored exactly in beforeAll;
fixture commits disable gpg signing. Wired into the periodic CI matrix,
paid-shard globs, eval scripts, touchfiles/E2E_TIERS
(codex-sol-scope-termination), and diff-based selection. Real-file
periodic-tier classification pins both codex E2Es out of the gate tier.
Free-tier test proves an explicit --model overrides the host default
through the real generation CLI.
This commit is contained in:
Garry Tan
2026-08-18 13:11:15 -07:00
parent 39a60588ed
commit 8229fdc489
10 changed files with 346 additions and 22 deletions
+17 -1
View File
@@ -10,6 +10,10 @@
*/
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
import {
PAID_TEST_GLOBS,
classifyPaidTestFile,
@@ -32,6 +36,7 @@ describe('paid test enumeration', () => {
expect(isPaidTestFile('test/skill-e2e-qa-workflow.test.ts')).toBe(true);
expect(isPaidTestFile('test/skill-llm-eval.test.ts')).toBe(true);
expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
expect(isPaidTestFile('test/codex-e2e-sol-scope.test.ts')).toBe(true);
expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
// Outside the globs: no dash, extra suffix, or a free test.
// 'test/skill-e2e.test.ts' is the DELETED pre-split monolith's name,
@@ -46,7 +51,7 @@ describe('paid test enumeration', () => {
const files = collectPaidTestFiles();
expect(files.length).toBeGreaterThan(0);
expect(files.every(isPaidTestFile)).toBe(true);
expect(PAID_TEST_GLOBS.length).toBe(5);
expect(PAID_TEST_GLOBS.length).toBe(6);
const shards = planPaidShards(files);
expect(shards.flat().sort()).toEqual([...files].sort());
@@ -87,6 +92,17 @@ describe('tier classification', () => {
expect(classifyPaidTestFile(noGuard, 'periodic').included).toBe(true);
expect(classifyPaidTestFile('', 'gate').included).toBe(true);
});
test('the REAL external-CLI test files classify as periodic-only', () => {
// Synthetic guard shapes above can drift from the actual files — the
// inert-demotion defect class. Pin the real sources: a guard-shape edit
// in either file that silently runs it in gate fails here.
for (const file of ['test/codex-e2e.test.ts', 'test/codex-e2e-sol-scope.test.ts']) {
const source = fs.readFileSync(path.join(ROOT, file), 'utf8');
expect(classifyPaidTestFile(source, 'gate').included, `${file} leaked into gate tier`).toBe(false);
expect(classifyPaidTestFile(source, 'periodic').included, `${file} dropped from periodic tier`).toBe(true);
}
});
});
describe('shard execution', () => {