mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-19 19:32:18 +02:00
feat: hermetic Codex runner hardening + Sol scope-termination E2E
The Codex E2E runner copies auth.json only (operator plugins, MCP servers, rules, and skills no longer leak into hermetic evals), pins CODEX_HOME to the temp dir, and supports per-run model, TOML overrides, and --ignore-user-config. New periodic E2E installs the FULL generated investigate skill on gpt-5.6-sol against a planted one-line bug with decoy TODOs: the fix must land inside the boundary (untracked files counted via git status --porcelain), decoys stay byte-identical, the regression oracle survives unweakened, nothing gets committed, all within 30 tool calls. The shared .agents tree is snapshotted and restored exactly in beforeAll; fixture commits disable gpg signing. Wired into the periodic CI matrix, paid-shard globs, eval scripts, touchfiles/E2E_TIERS (codex-sol-scope-termination), and diff-based selection. Real-file periodic-tier classification pins both codex E2Es out of the gate tier. Free-tier test proves an explicit --model overrides the host default through the real generation CLI.
This commit is contained in:
@@ -10,6 +10,10 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
import {
|
||||
PAID_TEST_GLOBS,
|
||||
classifyPaidTestFile,
|
||||
@@ -32,6 +36,7 @@ describe('paid test enumeration', () => {
|
||||
expect(isPaidTestFile('test/skill-e2e-qa-workflow.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-llm-eval.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/codex-e2e-sol-scope.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
|
||||
// Outside the globs: no dash, extra suffix, or a free test.
|
||||
// 'test/skill-e2e.test.ts' is the DELETED pre-split monolith's name,
|
||||
@@ -46,7 +51,7 @@ describe('paid test enumeration', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
expect(files.every(isPaidTestFile)).toBe(true);
|
||||
expect(PAID_TEST_GLOBS.length).toBe(5);
|
||||
expect(PAID_TEST_GLOBS.length).toBe(6);
|
||||
|
||||
const shards = planPaidShards(files);
|
||||
expect(shards.flat().sort()).toEqual([...files].sort());
|
||||
@@ -87,6 +92,17 @@ describe('tier classification', () => {
|
||||
expect(classifyPaidTestFile(noGuard, 'periodic').included).toBe(true);
|
||||
expect(classifyPaidTestFile('', 'gate').included).toBe(true);
|
||||
});
|
||||
|
||||
test('the REAL external-CLI test files classify as periodic-only', () => {
|
||||
// Synthetic guard shapes above can drift from the actual files — the
|
||||
// inert-demotion defect class. Pin the real sources: a guard-shape edit
|
||||
// in either file that silently runs it in gate fails here.
|
||||
for (const file of ['test/codex-e2e.test.ts', 'test/codex-e2e-sol-scope.test.ts']) {
|
||||
const source = fs.readFileSync(path.join(ROOT, file), 'utf8');
|
||||
expect(classifyPaidTestFile(source, 'gate').included, `${file} leaked into gate tier`).toBe(false);
|
||||
expect(classifyPaidTestFile(source, 'periodic').included, `${file} dropped from periodic tier`).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('shard execution', () => {
|
||||
|
||||
Reference in New Issue
Block a user