diff --git a/test/fixtures/codex-review-sandbox-unavailable-37158847998.json b/test/fixtures/codex-review-sandbox-unavailable-37158847998.json new file mode 100644 index 000000000..0223e886a --- /dev/null +++ b/test/fixtures/codex-review-sandbox-unavailable-37158847998.json @@ -0,0 +1,244 @@ +{ + "source": "https://github.com/garrytan/gstack/actions/runs/37158847998 (paid-slice-9, /codex review, recorded passed)", + "transcript": [ + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_01Mb4hJBn3WjkgygN9VgC9dK", + "name": "Bash", + "input": { + "command": "cat /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0/codex-SKILL.md", + "description": "Read the codex skill instructions" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_01Mb4hJBn3WjkgygN9VgC9dK", + "type": "tool_result", + "content": "[trimmed: skill text]", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_013KDMX84cHDfJGraY3MSeMy", + "name": "Bash", + "input": { + "command": "CODEX_BIN=$(command -v codex || echo \"\"); [ -z \"$CODEX_BIN\" ] && echo \"NOT_FOUND\" || echo \"FOUND: $CODEX_BIN\"; codex --version 2>&1 | head -1", + "description": "Check for codex binary and version" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_013KDMX84cHDfJGraY3MSeMy", + "type": "tool_result", + "content": "FOUND: /opt/codex/bin/codex\ncodex-cli 0.160.0", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_019TMYmxDs6QmmSrWV77BdkB", + "name": "Bash", + "input": { + "command": "ls -la ~/.claude/skills/gstack/bin/gstack-codex-probe ~/.claude/skills/gstack/bin/gstack-paths ~/.claude/skills/gstack/bin/gstack-review-log ~/.claude/skills/gstack/lib/outside-review-result.ts 2>&1; command -v bun; echo \"CODEX_THREAD_ID=${CODEX_THREAD_ID:-} CODEX_SANDBOX=${CODEX_SANDBOX:-} GSTACK_ACTIVE_HOST=${GSTACK_ACTIVE_HOST:-} CLAUDECODE=${CLAUDECODE:-}\"; echo \"CODEX_API_KEY set: $([ -n \"${CODEX_API_KEY:-}\" ] && echo yes || echo no); OPENAI_API_KEY set: $([ -n \"${OPENAI_API_KEY:-}\" ] && echo yes || echo no)\"; ls \"${CODEX_HOME:-$HOME/.codex}/auth.json\" 2>&1", + "description": "Check gstack helper scripts, harness markers, and codex auth signals" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_019TMYmxDs6QmmSrWV77BdkB", + "type": "tool_result", + "content": "-rwxr-xr-x 1 runner runner 24955 Oct 3 22:35 /github/home/.claude/skills/gstack/bin/gstack-codex-probe\n-rwxr-xr-x 1 runner runner 7380 Oct 3 22:35 /github/home/.claude/skills/gstack/bin/gstack-paths\n-rwxr-xr-x 1 runner runner 4008 Oct 3 22:35 /github/home/.claude/skills/gstack/bin/gstack-review-log\n-rw-r--r-- 1 runner runner 10979 Oct 3 22:35 /github/home/.claude/skills/gstack/lib/outside-review-result.ts\n/usr/local/bin/bun\nCODEX_THREAD_ID= CODEX_SANDBOX= GSTACK_ACTIVE_HOST= CLAUDECODE=1\nCODEX_API_KEY set: no; OPENAI_API_KEY set: no\n/github/home/.gstack-ci-codex/auth.json", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_01Qi418cvKMX8CqZsBNP86To", + "name": "Bash", + "input": { + "command": "git diff origin/main --stat 2>/dev/null | tail -1 || true; echo \"---local main---\"; git diff main...HEAD --stat; echo \"---diff---\"; git diff main...HEAD", + "description": "Show the branch diff against main" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_01Qi418cvKMX8CqZsBNP86To", + "type": "tool_result", + "content": "---local main---\n user_controller.rb | 14 ++++++++++++++\n 1 file changed, 14 insertions(+)\n---diff---\ndiff --git a/user_controller.rb b/user_controller.rb\nnew file mode 100644\nindex 0000000..6344e0f\n--- /dev/null\n+++ b/user_controller.rb\n@@ -0,0 +1,14 @@\n+class UserController < ApplicationController\n+ def show\n+ # SQL injection — interpolating user input directly into query\n+ @user = User.where(\"id = #{params[:id]}\").first\n+ render json: @user\n+ end\n+\n+ def promote\n+ # Bypasses ActiveRecord validations — update_column skips callbacks + validation\n+ @user = User.find(params[:id])\n+ @user.update_column(:role, 'admin')\n+ head :ok\n+ end\n+end", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_01HfjNsSdDDWLaY72ZPrAhYa", + "name": "Bash", + "input": { + "command": "cd /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0\n_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)\nsource ~/.claude/skills/gstack/bin/gstack-codex-probe\nif { [ -n \"${CODEX_THREAD_ID:-}\" ] || [ -n \"${CODEX_SANDBOX:-}\" ] || [ \"${GSTACK_ACTIVE_HOST:-}\" = codex ]; }; then\n echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2\n exit 78\nfi\nif ! _gstack_codex_auth_probe >/dev/null; then\n _gstack_codex_log_event \"codex_auth_failed\"\n echo \"AUTH_FAILED\"\nelif _gstack_codex_sandbox_preflight; then\n _gstack_codex_model_probe\nfi\n_gstack_codex_version_check\necho \"PROBE_EXIT=$?\"", + "description": "Run codex auth, sandbox, model and version probes", + "timeout": 120000 + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_01HfjNsSdDDWLaY72ZPrAhYa", + "type": "tool_result", + "content": "CODEX_SANDBOX: unavailable\nCodex outside review unavailable: Codex's sandbox could not start here (bwrap: Failed to make / slave: Permission denied). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.\nPROBE_EXIT=0", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_016szorKD1XPWENVS2ZmNQrs", + "name": "Bash", + "input": { + "command": "GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); echo \"GSTACK_STATE_ROOT=$GSTACK_STATE_ROOT\"\necho \"PLAN_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get PLAN_ROOT)\"\necho \"TMP_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get TMP_ROOT)\"", + "description": "Resolve gstack plan and temp roots" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_016szorKD1XPWENVS2ZmNQrs", + "type": "tool_result", + "content": "GSTACK_STATE_ROOT=/home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/gstack-hermetic-4342-Fa8WON/gstack-home\nPLAN_ROOT=/github/home/.claude/plans\nTMP_ROOT=/home/runner/.cache/gstack-paid-shard-yQbZbo/tmp", + "is_error": false + } + ] + } + }, + { + "type": "assistant", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "type": "tool_use", + "id": "toolu_01X2HQFyV9BstdEQBat9XWzV", + "name": "Bash", + "input": { + "command": "cd /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0\ncat > codex-output.md <<'EOF'\n# /codex review — feature/add-vuln vs main\n\n## Preflight\n\n- Step 0.4 — codex binary: `FOUND: /opt/codex/bin/codex` (codex-cli 0.160.0)\n- Step 0.5 — harness guard: clean (no `CODEX_THREAD_ID` / `CODEX_SANDBOX` / `GSTACK_ACTIVE_HOST=codex`)\n- Step 0.5 — auth probe: passed (`${CODEX_HOME}/auth.json` present)\n- Step 0.5 — sandbox preflight: **FAILED**\n\nProbe output (verbatim):\n\n```\nCODEX_SANDBOX: unavailable\nCodex outside review unavailable: Codex's sandbox could not start here (bwrap: Failed to make / slave: Permission denied). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.\n```\n\nModel probe and version check did not run because the sandbox preflight short-circuited them.\nPer the skill (\"If the output contains `CODEX_SANDBOX: unavailable`, stop\"), no `codex review`\ncommand was issued. No paid call was made. Tokens: 0 | Est. cost: $0.00\n\n## Scope that would have been reviewed\n\n`git diff main...HEAD --stat`:\n\n```\n user_controller.rb | 14 ++++++++++++++\n 1 file changed, 14 insertions(+)\n```\n\n## CODEX SAYS (code review):\n════════════════════════════════════════════════════════════\n(no output — Codex did not run; sandbox unavailable)\n════════════════════════════════════════════════════════════\nGATE: FAIL (fail-closed: Codex sandbox unavailable, no review ran — needs human attention)\n\nThe review did not complete. This is a fail-closed verdict, not a judgment on the code:\n`VERDICT: unavailable` maps to GATE: FAIL under gate check 2 (nothing was reviewed).\n\nRecommendation: Fix the Codex sandbox (enable unprivileged user namespaces, or set GSTACK_CODEX_NO_SANDBOX=1 and accept the full-access warning) and re-run `/codex review` before merging, because the diff adds a controller with string-interpolated SQL in `User.where(\"id = #{params[:id]}\")` and an unauthenticated `update_column(:role, 'admin')` path, both of which an outside review would almost certainly flag as P1, and shipping on a fail-closed gate would mean merging that with zero outside coverage.\nEOF\necho written; wc -l codex-output.md", + "description": "Write the fail-closed codex review result to the output file" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c", + "message": { + "content": [ + { + "tool_use_id": "toolu_01X2HQFyV9BstdEQBat9XWzV", + "type": "tool_result", + "content": "written\n39 codex-output.md", + "is_error": false + } + ] + } + } + ] +} diff --git a/test/helpers/outside-voice-evidence.ts b/test/helpers/outside-voice-evidence.ts index bf440f6d4..7c4cec7ed 100644 --- a/test/helpers/outside-voice-evidence.ts +++ b/test/helpers/outside-voice-evidence.ts @@ -197,6 +197,17 @@ export function outsideExecutionTranscript(executions: OutsideExecution[], provi })); } +/** + * Verdicts the shared validator (lib/outside-review-result.ts) printed for the + * /codex review command: the Bash execution that ran `codex review|exec` and + * piped it through the validator. A review that never ran prints none, or + * `unavailable`. + */ +export function codexReviewVerdicts(executions: OutsideExecution[]): string[] { + return executions.filter(({ command }) => outsideInvocation('codex').test(command) && command.includes('outside-review-result.ts')) + .flatMap(({ output }) => [...output.matchAll(/^VERDICT: ([a-z]+)\s*$/gm)].map(match => match[1]!)); +} + export function foundInvoiceAuthorizationDefect(executions: OutsideExecution[], provider: 'codex' | 'claude-code'): boolean { return executions.some(({ command, output, succeeded }) => succeeded && outsideInvocation(provider).test(command) diff --git a/test/outside-voice-evidence.test.ts b/test/outside-voice-evidence.test.ts index 2d0f6249c..e22e6ae95 100644 --- a/test/outside-voice-evidence.test.ts +++ b/test/outside-voice-evidence.test.ts @@ -3,7 +3,8 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { EvalCollector } from './helpers/eval-store'; -import { claudeOutsideExecutions, codexOutsideExecutions, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence'; +import { claudeOutsideExecutions, codexOutsideExecutions, codexReviewVerdicts, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence'; +import captured_codex_review_unavailable from './fixtures/codex-review-sandbox-unavailable-37158847998.json'; import captured_outside_background_ai from './fixtures/outside-background-ai.json'; import { parseNDJSON } from './helpers/session-runner'; import captured_outside_voice_async from './fixtures/outside-async-task-m-events.json'; @@ -36,6 +37,27 @@ describe('cross-harness live eval evidence', () => { }); }); +describe('/codex review execution evidence', () => { + const run = (command: string, output: string, isError = false) => [ + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'review', name: 'Bash', input: { command } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'review', content: output, is_error: isError }] } }, + ]; + const command = '_gstack_codex_timeout_wrapper 330 codex review --base main >"$TMPOUT" 2>"$TMPERR"\n' + + 'bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label \'Codex review\' --exit "$_CODEX_EXIT" --stderr "$TMPERR" structured "$TMPOUT"'; + + test('the captured sandbox-unavailable review (recorded passed in CI) has no executed verdict', () => { + expect(codexReviewVerdicts(claudeOutsideExecutions(captured_codex_review_unavailable.transcript))).toEqual([]); + }); + + test('only the validator verdict printed by the codex command counts', () => { + expect(codexReviewVerdicts(claudeOutsideExecutions(run(command, `[P1] SQL injection\nVERDICT: findings\nFINDINGS: P1`, true)))).toEqual(['findings']); + expect(codexReviewVerdicts(claudeOutsideExecutions(run(command, 'Codex outside review unavailable: sandbox\nVERDICT: unavailable', true)))).toEqual(['unavailable']); + expect(codexReviewVerdicts(claudeOutsideExecutions(run('cat > codex-output.md < { for (const provider of ['codex', 'claude-code'] as const) { test(`${provider} persists full successful and failed results before cleanup`, () => { diff --git a/test/skill-e2e-workflow.test.ts b/test/skill-e2e-workflow.test.ts index 6cd9672bc..8d566469d 100644 --- a/test/skill-e2e-workflow.test.ts +++ b/test/skill-e2e-workflow.test.ts @@ -15,6 +15,7 @@ import * as path from 'path'; import * as os from 'os'; import { extractSkillBody } from './helpers/skill-fixture'; import { createCoverageAuditFixture } from './fixtures/coverage-audit-fixture'; +import { claudeOutsideExecutions, codexReviewVerdicts } from './helpers/outside-voice-evidence'; import { validateCoverageAudit, type CoverageFile } from './helpers/coverage-audit'; import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt'; @@ -472,17 +473,17 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m }); logCost('/codex review', result); - recordE2E(evalCollector, '/codex review', 'Codex skill E2E', result); - expect(result.exitReason).toBe('success'); - - // Check that output file was created with review content + // A review counts only when Codex ran: the shared validator printed an executed + // verdict for the codex command, and the written gate is not fail-closed. + const verdicts = codexReviewVerdicts(claudeOutsideExecutions(result.transcript)); const outputPath = path.join(codexDir, 'codex-output.md'); - if (fs.existsSync(outputPath)) { - const output = fs.readFileSync(outputPath, 'utf-8'); - // Should contain the CODEX SAYS header or GATE verdict - const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex'); - expect(hasCodexOutput).toBe(true); - } + const output = fs.existsSync(outputPath) ? fs.readFileSync(outputPath, 'utf-8') : ''; + const executed = verdicts.some(verdict => ['findings', 'clean', 'unverified'].includes(verdict)); + const gated = /GATE: (?:PASS|FAIL \(\d+ critical|UNVERIFIED)/.test(output) && !/fail-closed/i.test(output); + recordE2E(evalCollector, '/codex review', 'Codex skill E2E', result, { passed: result.exitReason === 'success' && executed && gated }); + expect(result.exitReason).toBe('success'); + expect(executed, `no executed Codex review verdict in the codex command output (verdicts: ${verdicts.join(', ') || 'none'})`).toBe(true); + expect(gated, `codex-output.md has no executed GATE verdict:\n${output.slice(0, 1000)}`).toBe(true); }, CAPTURE_LONG_MS); });