From 446da35002a7f061c278e34c13d0ccf083178f00 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:39:58 +0000 Subject: [PATCH] E6: live Codex multi-block periodic case in the host-run Codex job test/codex-e2e-multiblock-live.test.ts runs ./setup --host codex once from the job's checkout into a fresh HOME and CODEX_HOME (auth.json copied from the CI login home), asserts codex debug prompt-input lists the gstack router as a real file, seeds one learning, and runs /learn's Show recent block through codex exec: that later block must root itself through its prelude, run the installed gstack-learnings-search and print the seeded learning; no block may report a missing install or an unresolved gstack helper path. The file joins CODEX_CI_FILES so the planner gives it to the host-run Codex job. paid-test-durations.json carries a 180 s packing estimate (refreshed by --write-durations after the first census) so the plan keeps every slice within max-parallel. Oracles live in test/helpers/codex-multiblock.ts with free replays in test/codex-multiblock.test.ts. Registration (E2E_TOUCHFILES, E2E_TIERS periodic, E2E_KINDS rule) is the integrator's. --- scripts/lib/paid-cases.ts | 1 + scripts/paid-test-durations.json | 1 + test/codex-e2e-multiblock-live.test.ts | 116 +++++++++++++++++++++++++ test/codex-multiblock.test.ts | 61 +++++++++++++ test/helpers/codex-multiblock.ts | 68 +++++++++++++++ 5 files changed, 247 insertions(+) create mode 100644 test/codex-e2e-multiblock-live.test.ts create mode 100644 test/codex-multiblock.test.ts create mode 100644 test/helpers/codex-multiblock.ts diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index 93b59280a..eae7a970b 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -263,6 +263,7 @@ export const CODEX_CI_FILES: readonly string[] = [ 'test/skill-e2e-outside-voice.test.ts', 'test/skill-e2e-outside-plan-disabled.test.ts', 'test/skill-e2e-safety-codex-boundary.test.ts', + 'test/codex-e2e-multiblock-live.test.ts', ]; /** diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index f206fd75f..d120102f4 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -122,6 +122,7 @@ "test/carve-section-loading-review.test.ts": 247902, "test/carve-section-loading-setup-gbrain.test.ts": 80994, "test/carve-section-loading-spec.test.ts": 176639, + "test/codex-e2e-multiblock-live.test.ts": 180000, "test/codex-e2e-recommendation-substance.test.ts": 17984, "test/codex-e2e-shared-libs.test.ts": 53442, "test/codex-e2e-sol-scope.test.ts": 37063, diff --git a/test/codex-e2e-multiblock-live.test.ts b/test/codex-e2e-multiblock-live.test.ts new file mode 100644 index 000000000..ccb4eebb4 --- /dev/null +++ b/test/codex-e2e-multiblock-live.test.ts @@ -0,0 +1,116 @@ +/** + * Live Codex multi-block check (E6, periodic, host-run Codex job). + * + * Codex runs every shell block of a skill in a fresh shell. This case runs + * `./setup --host codex` once from the job's own checkout into a fresh HOME + * and CODEX_HOME, then checks what a Codex user gets: + * 1. `codex debug prompt-input` lists the `gstack` router, and the router + * it names is a real file (Codex skips a symlinked SKILL.md); + * 2. `codex exec` running /learn's "Show recent" block, a later block that + * only its own prelude can root, runs the installed helper and prints the + * learning seeded before the run; no block reports a missing install or + * an unresolved gstack helper path. + * Auth is the CI login home's auth.json only (scopeCodexAccess hands this + * file the CLI on PATH and that home as CODEX_HOME). + */ +import { describe, test, afterAll, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawn, spawnSync } from 'child_process'; +import { CAPTURE_MS } from './helpers/eval-budgets'; +import { e2eTierEnabled } from './helpers/e2e-gate'; +import { hermeticChildEnv } from './helpers/hermetic-env'; +import { parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner'; +import { CODEX_EVAL_FINALIZE_MS, createCodexEvalCollector, runRecordedCodexEval } from './helpers/codex-eval'; +import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock'; +import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const CASE = 'codex-multiblock-live'; +const SETUP_MS = 300_000; +const CODEX_AVAILABLE = spawnSync('which', ['codex'], { timeout: 30_000 }).status === 0; +const ENABLED = e2eTierEnabled('periodic') && CODEX_AVAILABLE; +if (e2eTierEnabled('periodic') && !CODEX_AVAILABLE) process.stderr.write(`\n${CASE}: SKIPPED: codex binary not on PATH\n`); + +const collector = ENABLED ? createCodexEvalCollector('codex-e2e-multiblock-live') : null; +afterAll(async () => { await collector?.finalize(); }); + +/** `codex exec --json` with stdout/stderr in files (no pipe can outlive the process group), killed on abort. */ +function codexExec(args: string[], env: NodeJS.ProcessEnv, cwd: string, scratch: string, signal: AbortSignal): Promise { + const started = Date.now(); + const outFile = path.join(scratch, 'exec.jsonl'); + const errFile = path.join(scratch, 'exec.stderr'); + const out = fs.openSync(outFile, 'w'); + const err = fs.openSync(errFile, 'w'); + const proc = spawn('codex', args, { cwd, env, stdio: ['ignore', out, err], detached: true }); + fs.closeSync(out); + fs.closeSync(err); + const kill = () => { try { process.kill(-proc.pid!, 'SIGKILL'); } catch { /* already gone */ } }; + signal.addEventListener('abort', kill, { once: true }); + return new Promise(resolve => { + const finish = (exitCode: number) => { + signal.removeEventListener('abort', kill); + kill(); + const rawLines = fs.readFileSync(outFile, 'utf8').split('\n').filter(line => line.trim()); + resolve({ ...parseCodexJSONL(rawLines), rawLines, exitCode: signal.aborted ? 124 : exitCode, + durationMs: Date.now() - started, stderr: fs.readFileSync(errFile, 'utf8') }); + }; + proc.once('error', () => finish(1)); + proc.once('exit', (code, sig) => finish(code ?? (sig ? 128 + (os.constants.signals[sig] ?? 0) : 1))); + }); +} + +(ENABLED ? describe : describe.skip)('live Codex multi-block install (E6)', () => { + test(CASE, async () => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'e6-')); + try { + const home = path.join(scratch, 'home'); + const codexHome = path.join(home, '.codex'); + const repo = path.join(scratch, 'repo'); + // Codex will not create its sandbox helper aliases under its temp dir, so TMPDIR sits beside CODEX_HOME. + const tmp = path.join(home, 'tmp'); + for (const dir of [codexHome, repo, tmp]) fs.mkdirSync(dir, { recursive: true }); + const ciHome = process.env.CODEX_HOME ?? path.join(os.homedir(), '.codex'); + if (fs.existsSync(path.join(ciHome, 'auth.json'))) fs.copyFileSync(path.join(ciHome, 'auth.json'), path.join(codexHome, 'auth.json')); + const env = hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }); + + const setupStarted = Date.now(); + const setup = spawnSync('./setup', ['--host', 'codex'], { cwd: ROOT, env, encoding: 'utf8', timeout: SETUP_MS }); + console.log(`${CASE}: ./setup --host codex exit ${setup.status} in ${Math.round((Date.now() - setupStarted) / 1000)}s`); + expect(setup.status, `${setup.stdout.slice(-2000)}\n${setup.stderr.slice(-2000)}`).toBe(0); + + const promptInput = spawnSync('codex', ['debug', 'prompt-input'], { cwd: repo, env, encoding: 'utf8', timeout: 60_000 }); + expect(promptInput.status, promptInput.stderr).toBe(0); + const router = routerPathFromPromptInput(promptInput.stdout); + expect(router, 'codex debug prompt-input does not list the gstack router').toBe(path.join(codexHome, 'skills', 'gstack', 'SKILL.md')); + expect(fs.lstatSync(router!).isFile()).toBe(true); + + for (const args of [['init', '-q'], ['remote', 'add', 'origin', 'https://github.com/example/e6-fixture.git']]) { + expect(spawnSync('git', args, { cwd: repo, env, timeout: 30_000 }).status).toBe(0); + } + const seed = spawnSync(path.join(codexHome, 'skills', 'gstack', 'bin', 'gstack-learnings-log'), [JSON.stringify({ skill: 'learn', + type: 'pitfall', key: MULTIBLOCK_SENTINEL_KEY, insight: 'Seeded by the E6 live Codex check', confidence: 9, source: 'user-stated' })], + { cwd: repo, env, encoding: 'utf8', timeout: 30_000 }); + expect(seed.status, seed.stderr).toBe(0); + + const result = await runRecordedCodexEval({ + name: CASE, + suite: 'codex-e2e-multiblock-live', + budgetMs: CAPTURE_MS, + run: signal => codexExec(['exec', '--json', '-s', 'read-only', '--skip-git-repo-check', + '--model', process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL, + 'Use the gstack learn skill to show this project\'s recent learnings. Run its "Show recent" step, then list what it printed.'], + env, repo, scratch, signal), + validate: run => { + const problems = multiblockProblems(run.rawLines); + if (problems.length) throw new Error(problems.join('\n')); + }, + record: entry => collector?.addTest(entry), + }); + console.log(`${CASE}: ${result.toolCalls.length} commands, ${result.tokens} tokens, ${Math.round(result.durationMs / 1000)}s`); + } finally { + fs.rmSync(scratch, { recursive: true, force: true }); + } + }, SETUP_MS + 120_000 + CAPTURE_MS + CODEX_EVAL_FINALIZE_MS); +}); diff --git a/test/codex-multiblock.test.ts b/test/codex-multiblock.test.ts new file mode 100644 index 000000000..c96ca181b --- /dev/null +++ b/test/codex-multiblock.test.ts @@ -0,0 +1,61 @@ +/** + * Free replays for the live Codex multi-block oracles (E6): synthetic + * `codex exec --json` events in the shape codex-cli 0.160 writes (see + * test/fixtures/codex-sandbox/exec-json-healthy.jsonl) and a + * `codex debug prompt-input` skills block. + */ +import { describe, test, expect } from 'bun:test'; +import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock'; + +const PRELUDE = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;' + + '[ -d "$_r/bin" ]||_r=${CODEX_HOME:-~/.codex}/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;}'; +const SHOW_RECENT = `${PRELUDE}\nSLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG 2>/dev/null)\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20 2>/dev/null || echo "No learnings yet."`; +const LISTING = `LEARNINGS: 1 loaded (1 pitfall)\n\n## Pitfalls\n- [${MULTIBLOCK_SENTINEL_KEY}] (confidence: 9/10, user-stated, 2026-10-04)\n Seeded by the E6 live Codex check\n`; + +const command = (cmd: string, output: string, exitCode: number) => JSON.stringify({ type: 'item.completed', + item: { id: 'item_2', type: 'command_execution', command: `/bin/bash -lc ${JSON.stringify(cmd)}`, aggregated_output: output, exit_code: exitCode, status: exitCode === 0 ? 'completed' : 'failed' } }); +const run = (...lines: string[]) => ['{"type":"thread.started","thread_id":"t"}', ...lines, '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}']; +const preamble = command(`${PRELUDE}\nGSTACK_BIN=$GSTACK_ROOT/bin\n"$GSTACK_BIN/gstack-skill-start" --skill "learn"`, 'BRANCH: main\n', 0); + +describe('multiblockProblems', () => { + test('a later block rooted by its own prelude that prints the seeded learning passes', () => { + expect(multiblockProblems(run(preamble, command(SHOW_RECENT, LISTING, 0)))).toEqual([]); + }); + + test('a block that cannot find the install fails, even when another block succeeded', () => { + const missing = command(SHOW_RECENT, 'gstack: no install found (tried /root/.codex/skills/gstack). Fix: ./setup --host codex', 1); + const problems = multiblockProblems(run(preamble, missing, command(SHOW_RECENT, LISTING, 0))); + expect(problems).toHaveLength(1); + expect(problems[0]).toStartWith('a block could not find the install'); + }); + + test('the pre-prelude symptom (an empty root, /bin/gstack-slug) fails', () => { + const problems = multiblockProblems(run(command('SLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG)\n$GSTACK_ROOT/bin/gstack-learnings-search', + 'bash: line 1: /bin/gstack-slug: No such file or directory\nbash: line 2: /bin/gstack-learnings-search: No such file or directory\n', 127))); + expect(problems.some(p => p.startsWith('a gstack helper path did not resolve: /bin/gstack-slug: No such file or directory'))).toBe(true); + expect(problems.some(p => p.startsWith('no learnings-search block rooted by its own prelude printed the seeded learning'))).toBe(true); + }); + + test('a preamble-only run, a run without the seeded learning, or a hand-set root does not count', () => { + expect(multiblockProblems(run(preamble))).toEqual(['no block ran gstack-learnings-search (the skill\'s "Show recent" block)']); + expect(multiblockProblems(run(preamble, command(SHOW_RECENT, 'No learnings yet.\n', 0)))[0]).toContain('exit 0, sentinel absent'); + const handSet = command('GSTACK_ROOT=/home/u/.codex/skills/gstack\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20', LISTING, 0); + expect(multiblockProblems(run(preamble, handSet))[0]).toContain('GSTACK_ROOT set by hand'); + }); +}); + +describe('routerPathFromPromptInput', () => { + const promptInput = (entries: string[]) => JSON.stringify([{ type: 'message', role: 'developer', content: [{ type: 'input_text', + text: ['', '## Skills', '### Skill roots', '- `r0` = `/h/.codex/skills`', '- `r1` = `/h/.codex/skills/.system`', + '### Available skills', '- skill-creator: Create or update a Codex skill. (file: r1/skill-creator/SKILL.md)', ...entries].join('\n') }] }], null, 2); + + test('names the router file through the skill roots table', () => { + const listed = promptInput(['- gstack: Route a task to the right gstack skill. (gstack) (file: r0/gstack/SKILL.md)', + '- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']); + expect(routerPathFromPromptInput(`WARNING: proceeding without PATH aliases\n${listed}`)).toBe('/h/.codex/skills/gstack/SKILL.md'); + }); + + test('is null when Codex lists gstack skills but not the router', () => { + expect(routerPathFromPromptInput(promptInput(['- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']))).toBeNull(); + }); +}); diff --git a/test/helpers/codex-multiblock.ts b/test/helpers/codex-multiblock.ts new file mode 100644 index 000000000..eaef4edf9 --- /dev/null +++ b/test/helpers/codex-multiblock.ts @@ -0,0 +1,68 @@ +/** + * Oracles for the live Codex multi-block case (E6, + * test/codex-e2e-multiblock-live.test.ts). Codex runs every shell block in a + * fresh shell, so a gstack skill works there only when each later block + * resolves the install on its own (the env-var host prelude). The case + * installs gstack with `./setup --host codex` into a fresh CODEX_HOME, seeds + * one learning, and runs /learn's "Show recent" block through `codex exec`. + * Pure functions: the free test replays synthetic `exec --json` events. + */ + +export const MULTIBLOCK_SENTINEL_KEY = 'e6-multiblock-sentinel'; + +interface CommandEvent { command: string; output: string; exitCode: number | null } + +function completedCommands(rawLines: readonly string[]): CommandEvent[] { + const commands: CommandEvent[] = []; + for (const line of rawLines) { + try { + const event = JSON.parse(line); + if (event?.type !== 'item.completed' || event.item?.type !== 'command_execution') continue; + commands.push({ command: String(event.item.command ?? ''), output: String(event.item.aggregated_output ?? ''), + exitCode: typeof event.item.exit_code === 'number' ? event.item.exit_code : null }); + } catch { /* not an event line */ } + } + return commands; +} + +/** + * Problems with a live run: any block that could not find the install, a + * gstack helper path that did not resolve, or no later block (the skill's + * learnings-search block, which only the prelude can root) that ran the + * installed helper and printed the seeded learning. + */ +export function multiblockProblems(rawLines: readonly string[], sentinel = MULTIBLOCK_SENTINEL_KEY): string[] { + const commands = completedCommands(rawLines); + const problems: string[] = []; + for (const { command, output } of commands) { + if (output.includes('gstack: no install found')) problems.push(`a block could not find the install: ${output.trim().slice(0, 200)}`); + const unresolved = /\S*\/bin\/gstack-[\w-]+: (?:No such file or directory|command not found|not found)/.exec(output); + if (unresolved) problems.push(`a gstack helper path did not resolve: ${unresolved[0]} (in: ${command.slice(0, 120)})`); + } + const search = commands.filter(c => c.command.includes('gstack-learnings-search')); + if (search.length === 0) problems.push('no block ran gstack-learnings-search (the skill\'s "Show recent" block)'); + else if (!search.some(c => c.exitCode === 0 && c.output.includes(sentinel) && !/GSTACK_ROOT=["']?\//.test(c.command))) { + problems.push(`no learnings-search block rooted by its own prelude printed the seeded learning ${sentinel}: ` + + search.map(c => `exit ${c.exitCode}, ${c.output.includes(sentinel) ? 'sentinel present' : 'sentinel absent'}` + + `${/GSTACK_ROOT=["']?\//.test(c.command) ? ', GSTACK_ROOT set by hand' : ''}`).join('; ')); + } + return problems; +} + +/** + * The `gstack` router entry from `codex debug prompt-input` output (a JSON + * array of input items): the skill path it names, resolved through the skill + * roots table, or null when Codex does not list the router. + */ +export function routerPathFromPromptInput(promptInput: string): string | null { + let text = promptInput; + try { + const items = JSON.parse(promptInput.slice(promptInput.indexOf('['))); + text = (Array.isArray(items) ? items : []).flatMap((item: { content?: Array<{ text?: string }> }) => item.content ?? []) + .map(part => part.text ?? '').join('\n'); + } catch { /* scan the raw text */ } + const entry = /^- gstack: .*\(file: (r\d+)\/gstack\/SKILL\.md\)$/m.exec(text); + if (!entry) return null; + const root = new RegExp(`^- \`${entry[1]}\` = \`([^\`]+)\`$`, 'm').exec(text); + return root ? `${root[1]}/gstack/SKILL.md` : null; +}