I1: the /codex review eval passes only when Codex executed

Run 37158847998 recorded codex-review as passed while the sandbox preflight said unavailable and no review ran: the case only checked the session exit and an optional output file. It now requires the shared validator's executed verdict (findings, clean or unverified) in the codex command's own output and a written, non-fail-closed GATE.
2026-10-31 14:06:50 +01:00 · 2026-10-03 23:22:56 +00:00
1 parent 950cff358a
commit 34854c211b
@@ -0,0 +1,244 @@
+{
+  "source": "https://github.com/garrytan/gstack/actions/runs/37158847998 (paid-slice-9, /codex review, recorded passed)",
+  "transcript": [
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_01Mb4hJBn3WjkgygN9VgC9dK",
+            "name": "Bash",
+            "input": {
+              "command": "cat /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0/codex-SKILL.md",
+              "description": "Read the codex skill instructions"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_01Mb4hJBn3WjkgygN9VgC9dK",
+            "type": "tool_result",
+            "content": "[trimmed: skill text]",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_013KDMX84cHDfJGraY3MSeMy",
+            "name": "Bash",
+            "input": {
+              "command": "CODEX_BIN=$(command -v codex || echo \"\"); [ -z \"$CODEX_BIN\" ] && echo \"NOT_FOUND\" || echo \"FOUND: $CODEX_BIN\"; codex --version 2>&1 | head -1",
+              "description": "Check for codex binary and version"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_013KDMX84cHDfJGraY3MSeMy",
+            "type": "tool_result",
+            "content": "FOUND: /opt/codex/bin/codex\ncodex-cli 0.160.0",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_019TMYmxDs6QmmSrWV77BdkB",
+            "name": "Bash",
+            "input": {
+              "command": "ls -la ~/.claude/skills/gstack/bin/gstack-codex-probe ~/.claude/skills/gstack/bin/gstack-paths ~/.claude/skills/gstack/bin/gstack-review-log ~/.claude/skills/gstack/lib/outside-review-result.ts 2>&1; command -v bun; echo \"CODEX_THREAD_ID=${CODEX_THREAD_ID:-} CODEX_SANDBOX=${CODEX_SANDBOX:-} GSTACK_ACTIVE_HOST=${GSTACK_ACTIVE_HOST:-} CLAUDECODE=${CLAUDECODE:-}\"; echo \"CODEX_API_KEY set: $([ -n \"${CODEX_API_KEY:-}\" ] && echo yes || echo no); OPENAI_API_KEY set: $([ -n \"${OPENAI_API_KEY:-}\" ] && echo yes || echo no)\"; ls \"${CODEX_HOME:-$HOME/.codex}/auth.json\" 2>&1",
+              "description": "Check gstack helper scripts, harness markers, and codex auth signals"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_019TMYmxDs6QmmSrWV77BdkB",
+            "type": "tool_result",
+            "content": "-rwxr-xr-x 1 runner runner 24955 Oct  3 22:35 /github/home/.claude/skills/gstack/bin/gstack-codex-probe\n-rwxr-xr-x 1 runner runner  7380 Oct  3 22:35 /github/home/.claude/skills/gstack/bin/gstack-paths\n-rwxr-xr-x 1 runner runner  4008 Oct  3 22:35 /github/home/.claude/skills/gstack/bin/gstack-review-log\n-rw-r--r-- 1 runner runner 10979 Oct  3 22:35 /github/home/.claude/skills/gstack/lib/outside-review-result.ts\n/usr/local/bin/bun\nCODEX_THREAD_ID= CODEX_SANDBOX= GSTACK_ACTIVE_HOST= CLAUDECODE=1\nCODEX_API_KEY set: no; OPENAI_API_KEY set: no\n/github/home/.gstack-ci-codex/auth.json",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_01Qi418cvKMX8CqZsBNP86To",
+            "name": "Bash",
+            "input": {
+              "command": "git diff origin/main --stat 2>/dev/null | tail -1 || true; echo \"---local main---\"; git diff main...HEAD --stat; echo \"---diff---\"; git diff main...HEAD",
+              "description": "Show the branch diff against main"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_01Qi418cvKMX8CqZsBNP86To",
+            "type": "tool_result",
+            "content": "---local main---\n user_controller.rb | 14 ++++++++++++++\n 1 file changed, 14 insertions(+)\n---diff---\ndiff --git a/user_controller.rb b/user_controller.rb\nnew file mode 100644\nindex 0000000..6344e0f\n--- /dev/null\n+++ b/user_controller.rb\n@@ -0,0 +1,14 @@\n+class UserController < ApplicationController\n+  def show\n+    # SQL injection — interpolating user input directly into query\n+    @user = User.where(\"id = #{params[:id]}\").first\n+    render json: @user\n+  end\n+\n+  def promote\n+    # Bypasses ActiveRecord validations — update_column skips callbacks + validation\n+    @user = User.find(params[:id])\n+    @user.update_column(:role, 'admin')\n+    head :ok\n+  end\n+end",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_01HfjNsSdDDWLaY72ZPrAhYa",
+            "name": "Bash",
+            "input": {
+              "command": "cd /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0\n_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)\nsource ~/.claude/skills/gstack/bin/gstack-codex-probe\nif { [ -n \"${CODEX_THREAD_ID:-}\" ] || [ -n \"${CODEX_SANDBOX:-}\" ] || [ \"${GSTACK_ACTIVE_HOST:-}\" = codex ]; }; then\n  echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2\n  exit 78\nfi\nif ! _gstack_codex_auth_probe >/dev/null; then\n  _gstack_codex_log_event \"codex_auth_failed\"\n  echo \"AUTH_FAILED\"\nelif _gstack_codex_sandbox_preflight; then\n  _gstack_codex_model_probe\nfi\n_gstack_codex_version_check\necho \"PROBE_EXIT=$?\"",
+              "description": "Run codex auth, sandbox, model and version probes",
+              "timeout": 120000
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_01HfjNsSdDDWLaY72ZPrAhYa",
+            "type": "tool_result",
+            "content": "CODEX_SANDBOX: unavailable\nCodex outside review unavailable: Codex's sandbox could not start here (bwrap: Failed to make / slave: Permission denied). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.\nPROBE_EXIT=0",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_016szorKD1XPWENVS2ZmNQrs",
+            "name": "Bash",
+            "input": {
+              "command": "GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); echo \"GSTACK_STATE_ROOT=$GSTACK_STATE_ROOT\"\necho \"PLAN_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get PLAN_ROOT)\"\necho \"TMP_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get TMP_ROOT)\"",
+              "description": "Resolve gstack plan and temp roots"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_016szorKD1XPWENVS2ZmNQrs",
+            "type": "tool_result",
+            "content": "GSTACK_STATE_ROOT=/home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/gstack-hermetic-4342-Fa8WON/gstack-home\nPLAN_ROOT=/github/home/.claude/plans\nTMP_ROOT=/home/runner/.cache/gstack-paid-shard-yQbZbo/tmp",
+            "is_error": false
+          }
+        ]
+      }
+    },
+    {
+      "type": "assistant",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "type": "tool_use",
+            "id": "toolu_01X2HQFyV9BstdEQBat9XWzV",
+            "name": "Bash",
+            "input": {
+              "command": "cd /home/runner/.cache/gstack-paid-shard-yQbZbo/tmp/skill-e2e-codex-lYUSE0\ncat > codex-output.md <<'EOF'\n# /codex review — feature/add-vuln vs main\n\n## Preflight\n\n- Step 0.4 — codex binary: `FOUND: /opt/codex/bin/codex` (codex-cli 0.160.0)\n- Step 0.5 — harness guard: clean (no `CODEX_THREAD_ID` / `CODEX_SANDBOX` / `GSTACK_ACTIVE_HOST=codex`)\n- Step 0.5 — auth probe: passed (`${CODEX_HOME}/auth.json` present)\n- Step 0.5 — sandbox preflight: **FAILED**\n\nProbe output (verbatim):\n\n```\nCODEX_SANDBOX: unavailable\nCodex outside review unavailable: Codex's sandbox could not start here (bwrap: Failed to make / slave: Permission denied). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.\n```\n\nModel probe and version check did not run because the sandbox preflight short-circuited them.\nPer the skill (\"If the output contains `CODEX_SANDBOX: unavailable`, stop\"), no `codex review`\ncommand was issued. No paid call was made. Tokens: 0 | Est. cost: $0.00\n\n## Scope that would have been reviewed\n\n`git diff main...HEAD --stat`:\n\n```\n user_controller.rb | 14 ++++++++++++++\n 1 file changed, 14 insertions(+)\n```\n\n## CODEX SAYS (code review):\n════════════════════════════════════════════════════════════\n(no output — Codex did not run; sandbox unavailable)\n════════════════════════════════════════════════════════════\nGATE: FAIL (fail-closed: Codex sandbox unavailable, no review ran — needs human attention)\n\nThe review did not complete. This is a fail-closed verdict, not a judgment on the code:\n`VERDICT: unavailable` maps to GATE: FAIL under gate check 2 (nothing was reviewed).\n\nRecommendation: Fix the Codex sandbox (enable unprivileged user namespaces, or set GSTACK_CODEX_NO_SANDBOX=1 and accept the full-access warning) and re-run `/codex review` before merging, because the diff adds a controller with string-interpolated SQL in `User.where(\"id = #{params[:id]}\")` and an unauthenticated `update_column(:role, 'admin')` path, both of which an outside review would almost certainly flag as P1, and shipping on a fail-closed gate would mean merging that with zero outside coverage.\nEOF\necho written; wc -l codex-output.md",
+              "description": "Write the fail-closed codex review result to the output file"
+            },
+            "caller": {
+              "type": "direct"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "type": "user",
+      "session_id": "7836fde2-a2fc-4453-b81a-02f58922990c",
+      "message": {
+        "content": [
+          {
+            "tool_use_id": "toolu_01X2HQFyV9BstdEQBat9XWzV",
+            "type": "tool_result",
+            "content": "written\n39 codex-output.md",
+            "is_error": false
+          }
+        ]
+      }
+    }
+  ]
+}
@@ -197,6 +197,17 @@ export function outsideExecutionTranscript(executions: OutsideExecution[], provi
    }));
 }

+/**
+ * Verdicts the shared validator (lib/outside-review-result.ts) printed for the
+ * /codex review command: the Bash execution that ran `codex review|exec` and
+ * piped it through the validator. A review that never ran prints none, or
+ * `unavailable`.
+ */
+export function codexReviewVerdicts(executions: OutsideExecution[]): string[] {
+  return executions.filter(({ command }) => outsideInvocation('codex').test(command) && command.includes('outside-review-result.ts'))
+    .flatMap(({ output }) => [...output.matchAll(/^VERDICT: ([a-z]+)\s*$/gm)].map(match => match[1]!));
+}
+
 export function foundInvoiceAuthorizationDefect(executions: OutsideExecution[], provider: 'codex' | 'claude-code'): boolean {
  return executions.some(({ command, output, succeeded }) => succeeded
    && outsideInvocation(provider).test(command)
@@ -3,7 +3,8 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { EvalCollector } from './helpers/eval-store';
-import { claudeOutsideExecutions, codexOutsideExecutions, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence';
+import { claudeOutsideExecutions, codexOutsideExecutions, codexReviewVerdicts, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence';
+import captured_codex_review_unavailable from './fixtures/codex-review-sandbox-unavailable-37158847998.json';
 import captured_outside_background_ai from './fixtures/outside-background-ai.json';
 import { parseNDJSON } from './helpers/session-runner';
 import captured_outside_voice_async from './fixtures/outside-async-task-m-events.json';
@@ -36,6 +37,27 @@ describe('cross-harness live eval evidence', () => {
  });
 });

+describe('/codex review execution evidence', () => {
+  const run = (command: string, output: string, isError = false) => [
+    { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'review', name: 'Bash', input: { command } }] } },
+    { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'review', content: output, is_error: isError }] } },
+  ];
+  const command = '_gstack_codex_timeout_wrapper 330 codex review --base main >"$TMPOUT" 2>"$TMPERR"\n'
+    + 'bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label \'Codex review\' --exit "$_CODEX_EXIT" --stderr "$TMPERR" structured "$TMPOUT"';
+
+  test('the captured sandbox-unavailable review (recorded passed in CI) has no executed verdict', () => {
+    expect(codexReviewVerdicts(claudeOutsideExecutions(captured_codex_review_unavailable.transcript))).toEqual([]);
+  });
+
+  test('only the validator verdict printed by the codex command counts', () => {
+    expect(codexReviewVerdicts(claudeOutsideExecutions(run(command, `[P1] SQL injection\nVERDICT: findings\nFINDINGS: P1`, true)))).toEqual(['findings']);
+    expect(codexReviewVerdicts(claudeOutsideExecutions(run(command, 'Codex outside review unavailable: sandbox\nVERDICT: unavailable', true)))).toEqual(['unavailable']);
+    expect(codexReviewVerdicts(claudeOutsideExecutions(run('cat > codex-output.md <<EOF\nno `codex review` ran\nEOF', 'VERDICT: findings')))).toEqual([]);
+    expect(codexReviewVerdicts(claudeOutsideExecutions(run('codex review --base main', 'VERDICT: clean')))).toEqual([]);
+    expect(codexReviewVerdicts(claudeOutsideExecutions([{ type: 'assistant', message: { content: [{ type: 'text', text: 'VERDICT: findings' }] } }]))).toEqual([]);
+  });
+});
+
 describe('outside execution artifact retention', () => {
  for (const provider of ['codex', 'claude-code'] as const) {
    test(`${provider} persists full successful and failed results before cleanup`, () => {
@@ -15,6 +15,7 @@ import * as path from 'path';
 import * as os from 'os';
 import { extractSkillBody } from './helpers/skill-fixture';
 import { createCoverageAuditFixture } from './fixtures/coverage-audit-fixture';
+import { claudeOutsideExecutions, codexReviewVerdicts } from './helpers/outside-voice-evidence';
 import { validateCoverageAudit, type CoverageFile } from './helpers/coverage-audit';
 import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt';

@@ -472,17 +473,17 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m
    });

    logCost('/codex review', result);
-    recordE2E(evalCollector, '/codex review', 'Codex skill E2E', result);
-    expect(result.exitReason).toBe('success');
-
-    // Check that output file was created with review content
+    // A review counts only when Codex ran: the shared validator printed an executed
+    // verdict for the codex command, and the written gate is not fail-closed.
+    const verdicts = codexReviewVerdicts(claudeOutsideExecutions(result.transcript));
    const outputPath = path.join(codexDir, 'codex-output.md');
-    if (fs.existsSync(outputPath)) {
-      const output = fs.readFileSync(outputPath, 'utf-8');
-      // Should contain the CODEX SAYS header or GATE verdict
-      const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
-      expect(hasCodexOutput).toBe(true);
-    }
+    const output = fs.existsSync(outputPath) ? fs.readFileSync(outputPath, 'utf-8') : '';
+    const executed = verdicts.some(verdict => ['findings', 'clean', 'unverified'].includes(verdict));
+    const gated = /GATE: (?:PASS|FAIL \(\d+ critical|UNVERIFIED)/.test(output) && !/fail-closed/i.test(output);
+    recordE2E(evalCollector, '/codex review', 'Codex skill E2E', result, { passed: result.exitReason === 'success' && executed && gated });
+    expect(result.exitReason).toBe('success');
+    expect(executed, `no executed Codex review verdict in the codex command output (verdicts: ${verdicts.join(', ') || 'none'})`).toBe(true);
+    expect(gated, `codex-output.md has no executed GATE verdict:\n${output.slice(0, 1000)}`).toBe(true);
  }, CAPTURE_LONG_MS);
 });