mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
Merge lane wave/inv1-codex
This commit is contained in:
commit
93742a1ae2
33 files changed
+1603
-123
No files matched your search
+121
-15
@@ -5,6 +5,9 @@
|
||||
# Functions (all prefixed with _gstack_codex_ for namespace hygiene):
|
||||
# _gstack_codex_auth_probe — multi-signal auth check (env + file)
|
||||
# _gstack_codex_select_model — resolve + print the runtime model (#2914)
|
||||
# _gstack_codex_sandbox_mode — read-only, or full access only for GSTACK_CODEX_NO_SANDBOX=1
|
||||
# _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux)
|
||||
# _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
|
||||
# _gstack_codex_model_probe — round-trip probe of the selected model (#2477)
|
||||
# _gstack_codex_version_check — warn on known-bad Codex CLI versions
|
||||
# _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
|
||||
@@ -19,8 +22,9 @@
|
||||
# --- Auth probe -------------------------------------------------------------
|
||||
|
||||
_gstack_codex_auth_probe() {
|
||||
# Multi-signal: env vars OR auth file. Avoids false negatives for env-auth
|
||||
# users (CI, platform engineers) that a file-only check would reject.
|
||||
# Multi-signal: env vars OR auth file OR a custom provider's env_key. Avoids
|
||||
# false negatives for env-auth users (CI, platform engineers, self-hosted
|
||||
# OpenAI-compatible providers) that a file-only check would reject.
|
||||
local _codex_home="${CODEX_HOME:-$HOME/.codex}"
|
||||
# Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ]
|
||||
# alone allows whitespace; pair with a whitespace strip for robustness.
|
||||
@@ -31,7 +35,26 @@ _gstack_codex_auth_probe() {
|
||||
echo "AUTH_OK"
|
||||
return 0
|
||||
fi
|
||||
# Custom OpenAI-compatible providers (#2192): a [model_providers.<id>]
|
||||
# table names its credential variable in env_key. Only keys inside those
|
||||
# tables count, comments are ignored, and the name must be a plain shell
|
||||
# identifier before it is dereferenced.
|
||||
local _env_key _resolved _missing=""
|
||||
if [ -f "$_codex_home/config.toml" ]; then
|
||||
while IFS= read -r _env_key; do
|
||||
case "$_env_key" in ''|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac
|
||||
_resolved=$(printf '%s' "${!_env_key:-}" | tr -d '[:space:]')
|
||||
if [ -n "$_resolved" ]; then
|
||||
echo "AUTH_OK"
|
||||
return 0
|
||||
fi
|
||||
_missing="${_missing:+$_missing, }$_env_key"
|
||||
done < <(awk '/^[[:space:]]*#/ { next }
|
||||
/^[[:space:]]*\[/ { in_provider = ($0 ~ /^[[:space:]]*\[model_providers\.[^]]+\][[:space:]]*(#.*)?$/); next }
|
||||
in_provider && /^[[:space:]]*env_key[[:space:]]*=/ { v = $0; sub(/^[^=]*=[[:space:]]*/, "", v); sub(/[[:space:]]*#.*$/, "", v); gsub(/["\047[:space:]]/, "", v); print v }' "$_codex_home/config.toml" 2>/dev/null)
|
||||
fi
|
||||
echo "AUTH_FAILED"
|
||||
[ -n "$_missing" ] && echo "HINT: $_codex_home/config.toml names a custom provider key ($_missing), but it is not set in this shell. Export it, or run codex login." >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -65,6 +88,67 @@ _gstack_codex_select_model() {
|
||||
fi
|
||||
_GSTACK_CODEX_SEL_KIND="$_kind"
|
||||
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
||||
_gstack_codex_sandbox_mode
|
||||
}
|
||||
|
||||
# --- Sandbox (B1) -----------------------------------------------------------
|
||||
|
||||
_gstack_codex_sandbox_mode() {
|
||||
# Every gstack Codex command passes -s / sandbox_mode from this variable, so
|
||||
# one place decides it. Codex has no read-only mode without its sandbox, so
|
||||
# the escape hatch is full access: honored only for exactly 1, only from the
|
||||
# shell environment (never a dotenv file or config), and announced each use.
|
||||
if [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ]; then
|
||||
_GSTACK_CODEX_SANDBOX="danger-full-access"
|
||||
echo "WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust." >&2
|
||||
else
|
||||
_GSTACK_CODEX_SANDBOX="read-only"
|
||||
fi
|
||||
}
|
||||
|
||||
_gstack_codex_sandbox_preflight() {
|
||||
# Free check before the first paid call: `codex sandbox` starts the same
|
||||
# bubblewrap sandbox with no model. Only a recognized sandbox failure blocks
|
||||
# (exit 3, printed as missing coverage); success, a timeout, or an older CLI
|
||||
# without the subcommand defers to the post-run classifier. Linux only, and
|
||||
# skipped when the user turned the sandbox off. Fixtures:
|
||||
# test/fixtures/codex-sandbox/.
|
||||
[ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ] && return 0
|
||||
[ "$(uname -s 2>/dev/null)" = "Linux" ] || return 0
|
||||
local _out _code _line
|
||||
_out=$(_gstack_codex_timeout_wrapper 10 codex sandbox -c 'sandbox_mode="read-only"' true </dev/null 2>&1)
|
||||
_code=$?
|
||||
{ [ "$_code" -eq 0 ] || [ "$_code" -eq 124 ]; } && return 0
|
||||
_line=$(printf '%s\n' "$_out" | grep -m1 -iE '^[[:space:]]*bwrap: |bubblewrap is unavailable|landlock.{0,60}(fail|error|not supported|unsupported)|seccomp.{0,60}(fail|error)|user namespaces?.{0,80}(not (allowed|permitted|supported)|denied|disabled)' | sed 's/^[[:space:]]*//' | cut -c1-240)
|
||||
[ -n "$_line" ] || return 0
|
||||
echo "CODEX_SANDBOX: unavailable"
|
||||
echo "Codex outside review unavailable: Codex's sandbox could not start here ($_line). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1." >&2
|
||||
_gstack_codex_log_event "codex_sandbox_unavailable" 2>/dev/null || true
|
||||
return 3
|
||||
}
|
||||
|
||||
# --- First-use notice (#965) ------------------------------------------------
|
||||
|
||||
_gstack_codex_first_use_notice() {
|
||||
# Outside reviews are on by default and use whatever Codex login exists, so
|
||||
# the first one on a machine says where the code goes, with which account,
|
||||
# and how to turn reviews off. Shown once per state root; never blocks and
|
||||
# never prints a credential, only where it comes from.
|
||||
local _root _marker _codex_home="${CODEX_HOME:-$HOME/.codex}" _account _provider
|
||||
_root="$(. "${BASH_SOURCE[0]%/*}/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
|
||||
[ -n "$_root" ] || return 0
|
||||
_marker="$_root/.codex-review-notice-shown"
|
||||
[ -e "$_marker" ] && return 0
|
||||
if [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in CODEX_API_KEY"
|
||||
elif [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in OPENAI_API_KEY"
|
||||
elif grep -q '"auth_mode"[[:space:]]*:[[:space:]]*"chatgpt"' "$_codex_home/auth.json" 2>/dev/null; then _account="the ChatGPT account Codex is logged in with (see: codex login status)"
|
||||
elif [ -f "$_codex_home/auth.json" ]; then _account="the API key saved by codex login in $_codex_home/auth.json"
|
||||
else _account="the credentials your Codex config.toml provider names"
|
||||
fi
|
||||
_provider=$(sed -n 's/^[[:space:]]*model_provider[[:space:]]*=[[:space:]]*"\([A-Za-z0-9_.-]*\)".*/\1/p' "$_codex_home/config.toml" 2>/dev/null | head -1)
|
||||
echo "NOTICE: gstack outside reviews send the review prompt and code to Codex (provider: ${_provider:-openai}) using $_account. Reviews stay on; this notice shows once per machine. To turn them off: gstack-config set codex_reviews disabled" >&2
|
||||
mkdir -p "$_root" 2>/dev/null && : > "$_marker" 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Model round-trip probe (#2477) ------------------------------------------
|
||||
@@ -79,8 +163,9 @@ _gstack_codex_model_probe() {
|
||||
#
|
||||
# Contract:
|
||||
# MODEL_OK (exit 0) — round trip succeeded; cached 1h.
|
||||
# MODEL_UNUSABLE (exit 1) — deterministic model 400 or an invalid
|
||||
# model choice; hints printed. The 400 is cached 15 min: re-probing every
|
||||
# MODEL_UNUSABLE (exit 1) — deterministic model rejection (400, or a
|
||||
# 404 from a retired model or a wrong custom-provider base_url) or an
|
||||
# invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every
|
||||
# preflight charged the affected user a 30s round trip + real tokens per
|
||||
# review section, forever. Changing the model or config.toml changes the
|
||||
# cache signature and re-probes immediately; the short TTL covers
|
||||
@@ -89,9 +174,10 @@ _gstack_codex_model_probe() {
|
||||
# ENOENT, non-executable binary, missing vendor payload). Deterministic,
|
||||
# so fail-open is wrong: retrying never helps. Never cached — a reinstall
|
||||
# fixes it and must be picked up on the very next probe (#2742).
|
||||
# MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; FAIL-OPEN so a
|
||||
# slow network never wedges codex mode (the per-invocation Error
|
||||
# Handling entry still covers a later 400). Never cached.
|
||||
# MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still
|
||||
# runs so a slow network never wedges codex mode, but readiness is
|
||||
# unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE:
|
||||
# unverified) and the review's own classification decides. Never cached.
|
||||
#
|
||||
# Only call this AFTER _gstack_codex_auth_probe passes — probing without
|
||||
# auth just measures the auth failure again.
|
||||
@@ -162,13 +248,27 @@ _gstack_codex_model_probe() {
|
||||
echo "MODEL_OK"
|
||||
return 0
|
||||
fi
|
||||
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*400'; then
|
||||
# A model rejection is deterministic: a 400 ("is not supported", "requires a
|
||||
# newer version of Codex") or a 404. A retired model answers 404 "does not
|
||||
# exist or you do not have access to it", and a custom provider with a wrong
|
||||
# base_url answers a bare 404; neither is network luck, so neither may fall
|
||||
# through to the inconclusive branch below and report ready (#2843).
|
||||
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
|
||||
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
|
||||
echo "MODEL_UNUSABLE"
|
||||
printf '%s\n' "$_out" | grep -i "model" | head -3
|
||||
printf '%s\n' "$_out" | grep -iE "model|404" | head -3
|
||||
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
||||
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
||||
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
|
||||
if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then
|
||||
echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)."
|
||||
_gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
|
||||
elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
|
||||
echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
|
||||
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
|
||||
_gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
|
||||
else
|
||||
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
||||
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
# A CLI that cannot execute is deterministic, not transient: the fail-open
|
||||
@@ -190,7 +290,10 @@ _gstack_codex_model_probe() {
|
||||
fi
|
||||
# Timeout (124) or transient failure: fail-open with a warning. The probe
|
||||
# exists to catch the deterministic model 400, not to gate on network luck.
|
||||
echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — proceeding; if invocations fail with a model 400, see the codex skill's Error Handling entry."
|
||||
# Callers report CODEX_MODE: unverified, never ready; the exit stays 0 so
|
||||
# skills rendered before this marker keep running the review.
|
||||
_GSTACK_CODEX_PROBE_STATE="inconclusive"
|
||||
echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — CODEX_MODE: unverified; proceeding, and the review's own result is still checked. If invocations fail with a model 400, see the codex skill's Error Handling entry."
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -257,10 +360,12 @@ _gstack_codex_timeout_wrapper() {
|
||||
# mirror timeout(1)'s exit-124 contract. The watchdog's stdout is
|
||||
# detached so an early finish never blocks a caller's $(...) capture on
|
||||
# the orphaned sleep.
|
||||
"$@" &
|
||||
# Without job control a background command's stdin is /dev/null; keep the
|
||||
# caller's stdin so prompts fed with `codex exec -` arrive (#1674).
|
||||
"$@" <&0 &
|
||||
local _cmd_pid=$!
|
||||
(
|
||||
sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0
|
||||
sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0 # best-effort: the command already exited, nothing to stop
|
||||
_GSTACK_CODEX_WAITED=0
|
||||
while [ "$_GSTACK_CODEX_WAITED" -lt "$_grace" ] && kill -0 "$_cmd_pid" 2>/dev/null; do
|
||||
sleep 1
|
||||
@@ -288,7 +393,8 @@ _gstack_codex_log_event() {
|
||||
# Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl.
|
||||
# Gated on $_TEL != "off" (caller sets this from gstack-config).
|
||||
# Event types: codex_timeout, codex_auth_failed, codex_cli_missing,
|
||||
# codex_version_warning, codex_model_unusable.
|
||||
# codex_version_warning, codex_model_unusable, codex_sandbox_unavailable,
|
||||
# codex_model_retired, codex_cli_outdated.
|
||||
# Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt
|
||||
# content, env var values, or auth tokens.
|
||||
local _event="$1"
|
||||
|
||||
+24
-11
@@ -83,7 +83,7 @@ source ~/.claude/skills/gstack/bin/gstack-codex-probe
|
||||
if ! _gstack_codex_auth_probe >/dev/null; then
|
||||
_gstack_codex_log_event "codex_auth_failed"
|
||||
echo "AUTH_FAILED"
|
||||
else
|
||||
elif _gstack_codex_sandbox_preflight; then # free; Linux only
|
||||
_gstack_codex_model_probe # ~10s round trip on first run, cached 1h (#2477)
|
||||
fi
|
||||
_gstack_codex_version_check # warns if known-bad, non-blocking
|
||||
@@ -97,12 +97,18 @@ If the output contains `AUTH_FAILED`, stop and tell the user:
|
||||
If the output contains `MODEL_UNUSABLE`, stop — the selected model (named with its
|
||||
source on the `CODEX_MODEL:` line) is invalid or the account cannot use it. Relay the
|
||||
probe's HINT lines and
|
||||
follow the "Model not supported (HTTP 400)" recovery steps in
|
||||
follow the "Model not supported (HTTP 400 or 404)" recovery steps in
|
||||
`## Error Handling` below. Running the modes anyway just burns four
|
||||
invocations on the same 400 (#2477).
|
||||
|
||||
`MODEL_PROBE_INCONCLUSIVE` is non-blocking (timeout/transient network): pass
|
||||
the warning through and continue.
|
||||
If the output contains `CODEX_SANDBOX: unavailable`, stop: Codex's sandbox cannot
|
||||
start here, so every command it runs would fail and its review would read nothing.
|
||||
Relay the `Codex outside review unavailable: ...` line verbatim, including its fix.
|
||||
No paid call was made.
|
||||
|
||||
`MODEL_PROBE_INCONCLUSIVE` is non-blocking (timeout/transient network): report
|
||||
`CODEX_MODE: unverified`, pass the warning through and continue; the mode's own
|
||||
validator still decides the result.
|
||||
|
||||
If the version check printed a `WARN:` line, pass it through to the user verbatim
|
||||
(non-blocking — Codex may still work, but the user should upgrade).
|
||||
@@ -178,8 +184,8 @@ Every prompt sent to Codex MUST be prefixed with this boundary instruction:
|
||||
|
||||
This applies to Challenge mode (prompt) and Consult mode (persona prompt), and to the
|
||||
custom-instructions path of Review mode — all three use `codex exec`, which still takes
|
||||
a free-form prompt argument. It does **not** apply to the default scoped `codex review`
|
||||
call in Step 2A: that command is invoked with **no prompt argument at all** (see "Scope
|
||||
a free-form prompt (fed on stdin with `codex exec -`, so size and quoting never break it). It does **not** apply to the default scoped `codex review`
|
||||
call in Step 2A: that command is invoked with **no prompt at all** (see "Scope
|
||||
flags exclude the prompt argument" in the Review mode section), so there is nowhere to put the preamble. That
|
||||
is acceptable — `codex review --base` hands the model a pre-computed diff rather than
|
||||
turning it loose on the filesystem, so the rabbit-hole risk the boundary guards against
|
||||
@@ -287,11 +293,14 @@ If token count is not available, display: `Tokens: unknown`
|
||||
missing or wrong. A prompt-only `codex review` defaults to uncommitted changes, so a
|
||||
clean working tree reads as an empty review even when `<base>...HEAD` is large. Confirm
|
||||
`--base <base>` is actually on the command line.
|
||||
- **Model not supported (HTTP 400):** stderr shows
|
||||
- **Model not supported (HTTP 400 or 404):** stderr shows
|
||||
`The '<model>' model is not supported when using Codex with a ChatGPT account`
|
||||
(a `status: 400` / `invalid_request_error` naming a model). This is a
|
||||
model-entitlement problem, not an auth or network failure, and the auth probe
|
||||
cannot catch it. Recovery, in order:
|
||||
(a `status: 400` / `invalid_request_error` naming a model), or
|
||||
`404 Not Found: The model '<model>' does not exist or you do not have access to it`
|
||||
(a retired model; a bare 404 usually means a custom provider's `base_url` is wrong).
|
||||
`requires a newer version of Codex` means the CLI is too old: upgrade it instead.
|
||||
None of these is an auth or network failure, and the auth probe cannot catch them.
|
||||
Recovery, in order:
|
||||
1. Read the `CODEX_MODEL:` line: it names the model and where it came from.
|
||||
2. Fix that source: name another model for this request, update
|
||||
`GSTACK_CODEX_MODEL`, or change `model` (or `review_model`) in the Codex
|
||||
@@ -299,6 +308,9 @@ If token count is not available, display: `Tokens: unknown`
|
||||
them to a model the account can use.
|
||||
3. If Codex printed `[notice.model_migrations]`, use that replacement model.
|
||||
Never present this as a model stall or a PASS — it is a fail-closed gate result.
|
||||
- **`VERDICT: unavailable`:** the shared validator found the run did not execute (for
|
||||
example `Codex's sandbox could not start here`). Relay its line and fix verbatim; it is
|
||||
missing coverage, never a PASS. Details: `docs/troubleshooting.md` in the gstack checkout.
|
||||
- **Empty response:** If `$TMPRESP` is empty or doesn't exist, tell the user:
|
||||
"Codex returned no response. Check stderr for errors."
|
||||
- **Session resume failure:** If resume fails, delete the session file and start fresh.
|
||||
@@ -307,7 +319,8 @@ If token count is not available, display: `Tokens: unknown`
|
||||
|
||||
## Important Rules
|
||||
|
||||
- **Never modify files.** This skill is read-only. Codex runs in read-only sandbox mode.
|
||||
- **Never modify files.** This skill is read-only. Codex runs in read-only sandbox mode
|
||||
(full access only when the user exported `GSTACK_CODEX_NO_SANDBOX=1`; it warns on every use).
|
||||
- **Present output verbatim.** Do not truncate, summarize, or editorialize Codex's output
|
||||
before showing it. Show it in full inside the CODEX SAYS block.
|
||||
- **Add synthesis after, not instead of.** Any Claude commentary comes after the full output.
|
||||
|
||||
@@ -18,6 +18,7 @@ With focus (e.g., "security"):
|
||||
Review the changes on this branch against the base branch. Run `git diff origin/<base>` to see the diff. Focus specifically on SECURITY. Your job is to find every way an attacker could exploit this code. Think about injection vectors, auth bypasses, privilege escalation, data exposure, and timing attacks. Be adversarial."
|
||||
|
||||
2. Run codex exec with **JSONL output** to capture reasoning traces and tool calls.
|
||||
Replace `<prompt>` with the full prompt from step 1, verbatim and unescaped; Codex reads it on stdin.
|
||||
Use `timeout: 600000` on the Bash call (the tool's maximum) — the gate sits ABOVE the
|
||||
540s wrapper so the wrapper fires first, ends Codex, and prints its explicit stall message:
|
||||
|
||||
@@ -33,9 +34,15 @@ fi
|
||||
# Fix 1+2: wrap with timeout (gtimeout/timeout fallback chain via probe helper),
|
||||
# capture stderr to $TMPERR for auth error detection (was: 2>/dev/null).
|
||||
[ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_timeout_wrapper 540 codex exec "<prompt>" -C "$_REPO_ROOT" -s read-only {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json < /dev/null 2>"$TMPERR" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
# The prompt goes to Codex on stdin (codex exec -), verbatim: no shell quoting, no argv size limit.
|
||||
cat > "$_PROMPT_FILE" <<'CODEX_PROMPT_END'
|
||||
<prompt>
|
||||
CODEX_PROMPT_END
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$_PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
import sys, json
|
||||
turn_completed_count = 0
|
||||
turn_failed = False
|
||||
@@ -92,8 +99,14 @@ if grep -qiE "auth|login|unauthorized" "$TMPERR" 2>/dev/null; then
|
||||
echo "[codex auth error] $(head -1 "$TMPERR")"
|
||||
_gstack_codex_log_event "codex_auth_failed"
|
||||
fi
|
||||
bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label 'Codex challenge' --exit "$_CODEX_EXIT" --stderr "$TMPERR" --events "$TMPRESP.events" execution "$TMPRESP"
|
||||
rm -f "$TMPRESP" "$TMPRESP.events" "$_PROMPT_FILE"
|
||||
```
|
||||
|
||||
`VERDICT: unavailable` means the challenge did not run (its line names why, such as a
|
||||
sandbox that could not start): say so and present no findings from it. Otherwise
|
||||
present the output below.
|
||||
|
||||
This parses codex's JSONL events to extract reasoning traces, tool calls, and the final
|
||||
response. The `[codex thinking]` lines show what codex reasoned through before its answer.
|
||||
|
||||
|
||||
@@ -60,7 +60,8 @@ For non-plan consult prompts (user typed `/codex <question>`), still prepend the
|
||||
|
||||
<user's question>"
|
||||
|
||||
4. Run codex exec with **JSONL output** to capture reasoning traces. Use
|
||||
4. Run codex exec with **JSONL output** to capture reasoning traces. Replace `<prompt>`
|
||||
with the full prompt, verbatim and unescaped; Codex reads it on stdin. Use
|
||||
`timeout: 600000` on the Bash call (the tool's maximum; for both new and resumed
|
||||
sessions) — the gate sits ABOVE the 540s wrapper so the wrapper fires first, ends Codex,
|
||||
and prints its explicit stall message:
|
||||
@@ -78,7 +79,12 @@ fi
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
# Fix 1: wrap with timeout (gtimeout/timeout fallback chain via probe helper)
|
||||
_gstack_codex_timeout_wrapper 540 codex exec "<prompt>" -C "$_REPO_ROOT" -s read-only {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="medium"' {{CODEX_WEB_SEARCH_FLAG}} --json < /dev/null 2>"$TMPERR" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
# The prompt goes to Codex on stdin (codex exec -), verbatim: no shell quoting, no argv size limit.
|
||||
cat > "$_PROMPT_FILE" <<'CODEX_PROMPT_END'
|
||||
<prompt>
|
||||
CODEX_PROMPT_END
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="medium"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$_PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
import sys, json
|
||||
turn_completed_count = 0
|
||||
turn_failed = False
|
||||
@@ -133,6 +139,8 @@ elif [ "$_CODEX_EXIT" != "0" ]; then
|
||||
head -20 "$TMPERR" 2>/dev/null | sed 's/^/ /' || true
|
||||
_gstack_codex_log_event "codex_nonzero_exit" "consult:$_CODEX_EXIT"
|
||||
fi
|
||||
bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label 'Codex consult' --exit "$_CODEX_EXIT" --stderr "$TMPERR" --events "$TMPRESP.events" execution "$TMPRESP"
|
||||
rm -f "$_PROMPT_FILE"
|
||||
```
|
||||
|
||||
**Session-cost reality (#2387, measured):** every `codex exec` call — resumed
|
||||
@@ -155,7 +163,12 @@ cd "$_REPO_ROOT" || exit 1
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
# Fix 1: wrap with timeout (gtimeout/timeout fallback chain via probe helper)
|
||||
_gstack_codex_timeout_wrapper 540 codex exec resume <session-id> "<prompt>" -c 'sandbox_mode="read-only"' {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="medium"' {{CODEX_WEB_SEARCH_FLAG}} --json < /dev/null 2>"$TMPERR" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
# The prompt goes to Codex on stdin (codex exec -), verbatim: no shell quoting, no argv size limit.
|
||||
cat > "$_PROMPT_FILE" <<'CODEX_PROMPT_END'
|
||||
<prompt>
|
||||
CODEX_PROMPT_END
|
||||
_gstack_codex_timeout_wrapper 540 codex exec resume <session-id> - -c "sandbox_mode=\"${_GSTACK_CODEX_SANDBOX:?}\"" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="medium"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$_PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
<same python streaming parser as above, with flush=True on all print() calls>
|
||||
"
|
||||
# Fix 1: same hang detection pattern as new-session block
|
||||
@@ -171,8 +184,14 @@ elif [ "$_CODEX_EXIT" != "0" ]; then
|
||||
head -20 "$TMPERR" 2>/dev/null | sed 's/^/ /' || true
|
||||
_gstack_codex_log_event "codex_nonzero_exit" "consult-resume:$_CODEX_EXIT"
|
||||
fi
|
||||
bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label 'Codex consult' --exit "$_CODEX_EXIT" --stderr "$TMPERR" --events "$TMPRESP.events" execution "$TMPRESP"
|
||||
rm -f "$_PROMPT_FILE"
|
||||
```
|
||||
|
||||
`VERDICT: unavailable` from either block means Codex did not answer from a working
|
||||
environment (its line names why, such as a sandbox that could not start): relay that
|
||||
line and do not present the output as Codex's answer.
|
||||
|
||||
5. Capture session ID from the streamed output. The parser prints `SESSION_ID:<id>`
|
||||
from the `thread.started` event. Save it for follow-ups:
|
||||
```bash
|
||||
|
||||
@@ -25,6 +25,7 @@ instructions get their own path (below).
|
||||
1. Create temp files for output capture:
|
||||
```bash
|
||||
TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
TMPOUT=$(mktemp "$TMP_ROOT/codex-out-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
```
|
||||
|
||||
2. Run the review. No prompt argument — scope comes from `--base` (or `--commit <sha>`
|
||||
@@ -32,10 +33,11 @@ when reviewing a single commit, or `--uncommitted` for the working tree).
|
||||
|
||||
Use only one command path below. Remember its printed start token as CODEX_REVIEW_START before the review reads or receives the diff. Capture a new token only before a genuine rerun, never just to log fixes.
|
||||
|
||||
**Sandbox is pinned read-only via config override.** Top-level `codex review` has no
|
||||
**Sandbox is pinned via config override.** Top-level `codex review` has no
|
||||
`-s`/`--sandbox` flag (verified on 0.147.0: `codex review --help` lists none), so the
|
||||
read-only sandbox is set with `-c 'sandbox_mode="read-only"'` — the same form the
|
||||
consult resume path uses. Without it the call inherits the user's
|
||||
sandbox is set with `-c` from `_GSTACK_CODEX_SANDBOX`, which `_gstack_codex_select_model`
|
||||
sets so the effective setting is `sandbox_mode="read-only"` (full access only for
|
||||
`GSTACK_CODEX_NO_SANDBOX=1`, with a warning). Without it the call inherits the user's
|
||||
`~/.codex/config.toml` default, which on a trusted project can be WRITE access —
|
||||
contradicting this skill's read-only contract (#2496, #2524):
|
||||
|
||||
@@ -48,8 +50,9 @@ _gstack_codex_select_model review || exit 1
|
||||
# The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST
|
||||
# and a stall surfaces as a diagnosable exit 124 with an explicit message,
|
||||
# never as a silent harness kill that downstream reads as "no findings".
|
||||
_gstack_codex_timeout_wrapper 330 codex review --base <base> -c 'sandbox_mode="read-only"' {{CODEX_REVIEW_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} < /dev/null 2>"$TMPERR"
|
||||
_gstack_codex_timeout_wrapper 330 codex review --base <base> -c "sandbox_mode=\"${_GSTACK_CODEX_SANDBOX:?}\"" {{CODEX_REVIEW_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} < /dev/null >"$TMPOUT" 2>"$TMPERR"
|
||||
_CODEX_EXIT=$?
|
||||
cat "$TMPOUT"
|
||||
if [ "$_CODEX_EXIT" = "124" ]; then
|
||||
_gstack_codex_log_event "codex_timeout" "330"
|
||||
_gstack_codex_log_hang "review" "$(wc -c < "$TMPERR" 2>/dev/null || echo 0)"
|
||||
@@ -62,6 +65,7 @@ elif [ "$_CODEX_EXIT" != "0" ]; then
|
||||
head -20 "$TMPERR" 2>/dev/null | sed 's/^/ /' || true
|
||||
_gstack_codex_log_event "codex_nonzero_exit" "review:$_CODEX_EXIT"
|
||||
fi
|
||||
bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label 'Codex review' --exit "$_CODEX_EXIT" --stderr "$TMPERR" structured "$TMPOUT"
|
||||
```
|
||||
|
||||
If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`.
|
||||
@@ -92,14 +96,16 @@ _PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp
|
||||
git diff "<base>...HEAD" 2>/dev/null
|
||||
printf '\nDIFF_END\n'
|
||||
} > "$_PROMPT_FILE"
|
||||
_gstack_codex_timeout_wrapper 330 codex exec -s read-only "$(cat "$_PROMPT_FILE")" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} < /dev/null 2>"$TMPERR"
|
||||
_gstack_codex_timeout_wrapper 330 codex exec - -s "${_GSTACK_CODEX_SANDBOX:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPOUT" < "$_PROMPT_FILE" >"$TMPOUT.events" 2>"$TMPERR"
|
||||
_CODEX_EXIT=$?
|
||||
rm -f "$_PROMPT_FILE"
|
||||
cat "$TMPOUT"
|
||||
if [ "$_CODEX_EXIT" = "124" ]; then
|
||||
_gstack_codex_log_event "codex_timeout" "330"
|
||||
_gstack_codex_log_hang "review" "$(wc -c < "$TMPERR" 2>/dev/null || echo 0)"
|
||||
echo "Codex stalled past 5.5 minutes."
|
||||
fi
|
||||
bun ~/.claude/skills/gstack/lib/outside-review-result.ts --label 'Codex review' --exit "$_CODEX_EXIT" --stderr "$TMPERR" --events "$TMPOUT.events" structured "$TMPOUT"
|
||||
```
|
||||
|
||||
When you take this path, say so in the output header — `CODEX SAYS (code review — custom
|
||||
@@ -121,8 +127,9 @@ instead of the harness killing the call silently.
|
||||
grep "tokens used" "$TMPERR" 2>/dev/null || echo "tokens: unknown"
|
||||
```
|
||||
|
||||
4. Determine the gate verdict: **PASS**, **FAIL**, or **UNVERIFIED**.
|
||||
**The gate FAILS CLOSED**: a run that did not complete is a FAIL, and output the
|
||||
4. Determine the gate verdict: **PASS**, **FAIL**, or **UNVERIFIED**. The command
|
||||
above ends with the shared validator's `VERDICT:` and `FINDINGS:` lines; the checks
|
||||
below are what it computes. **The gate FAILS CLOSED**: a run that did not complete is a FAIL, and output the
|
||||
gate cannot check is UNVERIFIED, never a PASS. Work through these checks IN ORDER; the first
|
||||
match wins:
|
||||
|
||||
@@ -131,7 +138,11 @@ match wins:
|
||||
verified result). Expired auth, a bad flag, a timeout, or a model-entitlement
|
||||
400 all land here instead of masquerading as a clean pass.
|
||||
2. The captured review output is empty or whitespace-only → **GATE: FAIL**
|
||||
(fail-closed: empty output — nothing was reviewed).
|
||||
(fail-closed: empty output — nothing was reviewed). Any other
|
||||
`VERDICT: unavailable` is also **GATE: FAIL**: Codex's sandbox could not
|
||||
start, every command it ran failed, or it said it could not read the diff.
|
||||
Relay the validator's `unavailable:` line; a "no findings" written after
|
||||
that is not a review.
|
||||
3. The output contains `[P0]` or `[P1]` (or codex's native unbracketed `P0:` /
|
||||
`P1:` severity labels) → **GATE: FAIL** (N critical findings). Codex's own
|
||||
review rubric treats P0 as blocking; this gate does too.
|
||||
@@ -145,7 +156,9 @@ match wins:
|
||||
5. Severity tags are present and none is P0/P1 (only P2/P3 advisory) →
|
||||
**GATE: PASS**.
|
||||
|
||||
There is no default branch: PASS is only reachable through check 5. For FAIL
|
||||
Check 3 is `VERDICT: findings`; check 4 is `VERDICT: unverified`, or
|
||||
`VERDICT: clean` with `FINDINGS: none`; check 5 is `VERDICT: clean` with
|
||||
`FINDINGS: P2` or `P3`. There is no default branch: PASS is only reachable through check 5. For FAIL
|
||||
via checks 1 or 2, say explicitly that the review did not complete. For
|
||||
UNVERIFIED, tell the user to read Codex's verdict above and decide; it is
|
||||
not a blocker by itself and not a pass.
|
||||
@@ -213,12 +226,13 @@ Substitute: TIMESTAMP (ISO 8601), STATUS ("clean" if PASS, "issues_found" if FAI
|
||||
"unverified" if UNVERIFIED), GATE ("pass", "fail" or "unverified" — fail-closed
|
||||
verdicts log as "fail"), findings (count of [P0]–[P3] markers; 0 for fail-closed
|
||||
and UNVERIFIED runs),
|
||||
findings_fixed (count of findings that were addressed/fixed before shipping).
|
||||
findings_fixed (count of findings that were addressed/fixed before shipping). If the
|
||||
`WARNING: GSTACK_CODEX_NO_SANDBOX=1` line printed, add `"sandbox":"danger-full-access"` to the record.
|
||||
CODEX_REVIEW_START is the original token from the command path that ran. COMPLETED is true only when the review completed with coverage of the branch diff and current working-tree changes; timeout, failure, refusal, or missing coverage is false. A limited `--commit`/`--uncommitted` review, or a committed-only custom prompt that omitted dirty/untracked source, does not establish whole-branch coverage: log completed false and explain the limitation. CONVERGED is true only for a completed pass with zero edits. Fixes stay stale until a genuine rerun reads the updated diff with a new start token. These evidence fields do not change the gate verdict above.
|
||||
|
||||
8. Clean up temp files:
|
||||
```bash
|
||||
rm -f "$TMPERR"
|
||||
rm -f "$TMPERR" "$TMPOUT" "$TMPOUT.events"
|
||||
```
|
||||
|
||||
---
|
||||
@@ -0,0 +1,208 @@
|
||||
# Troubleshooting gstack messages
|
||||
|
||||
Find the message you saw, then follow its fix. Every check gstack runs ends in
|
||||
one of three states:
|
||||
|
||||
- **ran**: the check executed. It may still have found problems.
|
||||
- **not run**: gstack chose not to run it (you turned it off, or nothing applied).
|
||||
- **unavailable**: gstack tried and could not run it. This is missing coverage,
|
||||
never a pass.
|
||||
|
||||
A message that is not `ran` has this shape:
|
||||
|
||||
```
|
||||
<check>: not run (<reason>). Fix: <command>
|
||||
<check> unavailable: <reason> (<detail>). No review ran; this is missing coverage, not a pass. Fix: <command>
|
||||
<check>: ran, verdict unverified (<reason>). Fix: read the output above
|
||||
```
|
||||
|
||||
Search this page for the words after `unavailable:` or `not run (`. Each
|
||||
section has a stable link anchor; the reason codes and anchors come from
|
||||
`lib/gate-outcomes.ts`, and a free test fails if a code has no section here.
|
||||
|
||||
---
|
||||
|
||||
## Outside reviews (Codex and Claude Code)
|
||||
|
||||
Outside reviews send your diff, plan or question to a second AI provider. Their
|
||||
verdict comes from `lib/outside-review-result.ts`, which prints
|
||||
`VERDICT: clean|findings|unverified|unavailable`. A `findings` verdict with a
|
||||
P0 or P1 finding blocks exactly like a native P0/P1. `unverified` and
|
||||
`unavailable` are missing coverage: /ship and /review continue, show the gap in
|
||||
the readiness dashboard and the PR body, and never count it as a pass.
|
||||
|
||||
<a id="codex-sandbox-unavailable"></a>
|
||||
### `Codex outside review unavailable: Codex's sandbox could not start here (...)`
|
||||
|
||||
**Meaning.** Codex runs every command inside a Linux sandbox (bubblewrap). In
|
||||
many containers and devcontainers the kernel does not allow unprivileged user
|
||||
namespaces, so the sandbox cannot start and every command Codex tries fails.
|
||||
gstack reports this instead of trusting a review that read nothing.
|
||||
`CODEX_MODE: sandbox_unavailable` is the same condition found by the free
|
||||
preflight before any paid call.
|
||||
|
||||
**What is kept.** Nothing was sent for review, or the review was discarded. Your
|
||||
code and files are unchanged.
|
||||
|
||||
**Fix.** Enable unprivileged user namespaces for the container (for Docker, a
|
||||
seccomp profile that allows them), or, inside a container you trust, run
|
||||
Codex without its sandbox for this shell:
|
||||
|
||||
```bash
|
||||
export GSTACK_CODEX_NO_SANDBOX=1
|
||||
```
|
||||
|
||||
**Expected result.** With namespaces enabled, the next review prints
|
||||
`OUTSIDE_STATUS: completed`. With `GSTACK_CODEX_NO_SANDBOX=1`, every review
|
||||
prints `WARNING: GSTACK_CODEX_NO_SANDBOX=1: ...` because Codex can then read
|
||||
and write anything your user can. Only the exact value `1` works, and only from
|
||||
your shell environment.
|
||||
|
||||
<a id="outside-review-commands-failed"></a>
|
||||
### `... outside review unavailable: the reviewer could not run commands or read the diff (...)`
|
||||
|
||||
**Meaning.** The reviewer tried to run commands but none succeeded, or its
|
||||
answer says it could not run commands or read the diff. A "no issues found"
|
||||
written after that is not a review.
|
||||
|
||||
**What is kept.** Nothing; the answer is shown above but not counted.
|
||||
|
||||
**Fix.** Read the reviewer's stderr above, repair what it names, and re-run.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed` with a `VERDICT:` line.
|
||||
|
||||
<a id="outside-review-execution-failed"></a>
|
||||
### `... outside review unavailable: the reviewer process failed (exit N: ...)`
|
||||
|
||||
**Meaning.** The provider CLI exited non-zero. The first stderr line is in the
|
||||
parentheses.
|
||||
|
||||
**What is kept.** Partial output is shown above; it is not counted.
|
||||
|
||||
**Fix.** Repair the cause in the provider diagnosis (log in again, fix the
|
||||
model, check the network), then re-run.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed`.
|
||||
|
||||
<a id="outside-review-timeout"></a>
|
||||
### `... outside review unavailable: the reviewer hit its time limit and was stopped (exit 124)`
|
||||
|
||||
**Meaning.** The provider did not finish within its deadline and was stopped.
|
||||
|
||||
**What is kept.** Partial output is shown above; it is not counted.
|
||||
|
||||
**Fix.** Re-run with a smaller scope, or check the provider's status and your
|
||||
network.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed`.
|
||||
|
||||
<a id="outside-review-empty-response"></a>
|
||||
### `... outside review unavailable: the reviewer returned no response`
|
||||
|
||||
**Meaning.** The provider exited successfully but returned nothing.
|
||||
|
||||
**Fix.** Read the stderr above, then re-run.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed`.
|
||||
|
||||
<a id="outside-review-refused"></a>
|
||||
### `... outside review unavailable: the reviewer declined to review`
|
||||
|
||||
**Meaning.** The provider answered with a refusal instead of a review.
|
||||
|
||||
**Fix.** Re-run; if it declines again, rely on the native review.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed`.
|
||||
|
||||
<a id="outside-review-missing-markers"></a>
|
||||
### `... outside review unavailable: the response lacks the markers this gate requires (...)`
|
||||
|
||||
**Meaning.** The answer did not contain what the gate checks: a
|
||||
`Recommendation: ... because ...` line, a `SCORE:`/`AMBIGUITIES:` pair, or
|
||||
severity tags.
|
||||
|
||||
**Fix.** Re-run the review.
|
||||
|
||||
**Expected result.** `OUTSIDE_STATUS: completed`.
|
||||
|
||||
<a id="outside-review-unverified"></a>
|
||||
### `... outside review: ran, verdict unverified (...)` / `OUTSIDE_STATUS: unverified` / `GATE: UNVERIFIED`
|
||||
|
||||
**Meaning.** The review completed but tagged nothing and gave no explicit
|
||||
no-findings conclusion, so no pass or fail can be read from it.
|
||||
|
||||
**What is kept.** The full answer is shown above.
|
||||
|
||||
**Fix.** Read the output above and decide. It is not a pass, and /ship and
|
||||
/review list it as missing coverage.
|
||||
|
||||
<a id="codex-model-unusable"></a>
|
||||
### `CODEX_MODE: model_unusable` / `MODEL_UNUSABLE`
|
||||
|
||||
**Meaning.** Codex rejected the selected model: the account cannot use it
|
||||
(HTTP 400), the model is retired, or a custom provider's `base_url` is wrong
|
||||
(HTTP 404). The `CODEX_MODEL:` line names the model and where it came from.
|
||||
|
||||
**Fix.** Choose a model your account can use, or correct the provider:
|
||||
|
||||
```bash
|
||||
export GSTACK_CODEX_MODEL=<supported-model>
|
||||
# or edit model / base_url in ${CODEX_HOME:-~/.codex}/config.toml
|
||||
```
|
||||
|
||||
**Expected result.** `CODEX_MODE: ready`.
|
||||
|
||||
<a id="codex-mode-unverified"></a>
|
||||
### `CODEX_MODE: unverified` / `MODEL_PROBE_INCONCLUSIVE`
|
||||
|
||||
**Meaning.** The short model check timed out or hit a network error, so gstack
|
||||
could not confirm the model works. The review still runs and its own result is
|
||||
checked.
|
||||
|
||||
**Fix.** Nothing now. If the review then fails, its message says why.
|
||||
|
||||
<a id="codex-auth-failed"></a>
|
||||
### `CODEX_MODE: not_authed` / `AUTH_FAILED`
|
||||
|
||||
**Meaning.** No Codex credentials were found: no `CODEX_API_KEY`, no
|
||||
`OPENAI_API_KEY`, no `auth.json`, and no set environment variable named by a
|
||||
custom provider's `env_key` in `config.toml`.
|
||||
|
||||
**Fix.**
|
||||
|
||||
```bash
|
||||
codex login
|
||||
```
|
||||
|
||||
or export the variable your provider's `env_key` names.
|
||||
|
||||
**Expected result.** `CODEX_MODE: ready`.
|
||||
|
||||
<a id="outside-review-disabled"></a>
|
||||
### `Codex review skipped (codex_reviews disabled)` / `CODEX_MODE: disabled`
|
||||
|
||||
**Meaning.** You turned outside reviews off. This is `not run`, never a pass and
|
||||
never an outage.
|
||||
|
||||
**Fix.** To turn them back on:
|
||||
|
||||
```bash
|
||||
gstack-config set codex_reviews enabled
|
||||
```
|
||||
|
||||
<a id="codex-review-notice"></a>
|
||||
### `NOTICE: gstack outside reviews send the review prompt and code to Codex (...) using ...`
|
||||
|
||||
**Meaning.** Outside reviews are on by default. The first one on a machine
|
||||
says which provider receives your prompt and code and which login or key pays
|
||||
for it. It shows once (gstack records `.codex-review-notice-shown` in its state
|
||||
directory) and never blocks the review.
|
||||
|
||||
**Fix.** Nothing, if that is what you want. To stop sending code to Codex:
|
||||
|
||||
```bash
|
||||
gstack-config set codex_reviews disabled
|
||||
```
|
||||
|
||||
**Expected result.** Later reviews print `Codex review skipped (codex_reviews disabled)`
|
||||
or `CODEX_MODE: disabled`, and /ship and /review show the outside review as not run.
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* INV-1 gate outcome contract: every gate or outside review ends as
|
||||
* `ran` (with findings or a verdict), `not_run` (gstack chose not to run it) or
|
||||
* `unavailable` (it tried and could not). Each reason code maps to one state,
|
||||
* one stable anchor in docs/troubleshooting.md and one fix action; the
|
||||
* validator prints these lines and test/troubleshooting-anchors.test.ts checks
|
||||
* every anchor exists. Add a row here before printing a new reason anywhere.
|
||||
*/
|
||||
export type GateState = 'ran' | 'not_run' | 'unavailable';
|
||||
|
||||
export interface GateOutcome {
|
||||
state: GateState;
|
||||
/** Stable `<a id>` in docs/troubleshooting.md. */
|
||||
anchor: string;
|
||||
/** What happened, in plain words; `<detail>` adds the real value (exit code, stderr line). */
|
||||
summary: string;
|
||||
fix: string;
|
||||
}
|
||||
|
||||
export const GATE_OUTCOMES = {
|
||||
sandbox_unavailable: {
|
||||
state: 'unavailable', anchor: 'codex-sandbox-unavailable',
|
||||
summary: "Codex's sandbox could not start here",
|
||||
fix: 'enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1',
|
||||
},
|
||||
commands_failed: {
|
||||
state: 'unavailable', anchor: 'outside-review-commands-failed',
|
||||
summary: 'the reviewer could not run commands or read the diff',
|
||||
fix: 'read the reviewer stderr above, repair the environment it names, then re-run the review',
|
||||
},
|
||||
execution_failed: {
|
||||
state: 'unavailable', anchor: 'outside-review-execution-failed',
|
||||
summary: 'the reviewer process failed',
|
||||
fix: 'read the provider diagnosis above (auth, model, network), repair it, then re-run the review',
|
||||
},
|
||||
timeout: {
|
||||
state: 'unavailable', anchor: 'outside-review-timeout',
|
||||
summary: 'the reviewer hit its time limit and was stopped',
|
||||
fix: 're-run the review with a smaller scope, or check the provider status and network',
|
||||
},
|
||||
empty_response: {
|
||||
state: 'unavailable', anchor: 'outside-review-empty-response',
|
||||
summary: 'the reviewer returned no response',
|
||||
fix: 'read the reviewer stderr above, then re-run the review',
|
||||
},
|
||||
review_refused: {
|
||||
state: 'unavailable', anchor: 'outside-review-refused',
|
||||
summary: 'the reviewer declined to review',
|
||||
fix: 're-run the review; if it declines again, review the change natively',
|
||||
},
|
||||
missing_markers: {
|
||||
state: 'unavailable', anchor: 'outside-review-missing-markers',
|
||||
summary: 'the response lacks the markers this gate requires',
|
||||
fix: 're-run the review; a response without its required markers never counts as a pass',
|
||||
},
|
||||
untagged_review: {
|
||||
state: 'ran', anchor: 'outside-review-unverified',
|
||||
summary: 'the review completed without severity tags or an explicit no-findings line',
|
||||
fix: 'read the output above and decide; it is not a pass',
|
||||
},
|
||||
model_unusable: {
|
||||
state: 'unavailable', anchor: 'codex-model-unusable',
|
||||
summary: 'Codex could not use the selected model',
|
||||
fix: 'set GSTACK_CODEX_MODEL=<supported-model> or fix model (and base_url for a custom provider) in your Codex config.toml',
|
||||
},
|
||||
probe_inconclusive: {
|
||||
state: 'ran', anchor: 'codex-mode-unverified',
|
||||
summary: 'the Codex model check timed out or hit a network error, so readiness is unverified',
|
||||
fix: 'nothing to do now; the review itself is still checked, and a failure there is reported',
|
||||
},
|
||||
auth_failed: {
|
||||
state: 'unavailable', anchor: 'codex-auth-failed',
|
||||
summary: 'no Codex credentials were found',
|
||||
fix: 'run codex login, or set the API key your Codex config.toml provider names in env_key',
|
||||
},
|
||||
disabled: {
|
||||
state: 'not_run', anchor: 'outside-review-disabled',
|
||||
summary: 'outside reviews are turned off (codex_reviews=disabled)',
|
||||
fix: 'gstack-config set codex_reviews enabled',
|
||||
},
|
||||
} as const satisfies Record<string, GateOutcome>;
|
||||
|
||||
export type GateReason = keyof typeof GATE_OUTCOMES;
|
||||
|
||||
/** One user-facing line for an outcome that is not a clean `ran`. */
|
||||
export function gateOutcomeLine(gate: string, reason: GateReason, detail?: string): string {
|
||||
const outcome: GateOutcome = GATE_OUTCOMES[reason];
|
||||
const what = detail ? `${outcome.summary} (${detail})` : outcome.summary;
|
||||
if (outcome.state === 'not_run') return `${gate}: not run (${what}). Fix: ${outcome.fix}.`;
|
||||
if (outcome.state === 'ran') return `${gate}: ran, verdict unverified (${what}). Fix: ${outcome.fix}.`;
|
||||
return `${gate} unavailable: ${what}. No review ran; this is missing coverage, not a pass. Fix: ${outcome.fix}.`;
|
||||
}
|
||||
+181
-31
@@ -1,38 +1,188 @@
|
||||
/** Review-specific completion evidence, separate from provider transport success. */
|
||||
export type OutsideGate = 'review' | 'structured' | 'spec';
|
||||
export function validateOutsideReview(text: string, gate: OutsideGate): { completed: boolean; reason?: string; score?: number; gate?: 'pass' | 'fail' } {
|
||||
if (!text.trim()) return { completed: false, reason: 'empty response' };
|
||||
// "I cannot find any issues" is a legitimate clean conclusion. Match a
|
||||
// refused/unavailable review, not every use of a negative auxiliary verb.
|
||||
if (/\b(?:(?:I (?:cannot|can't|won't|will not|am unable to)|I'm unable to)\s+(?:review|analy[sz]e|evaluate|assess|inspect|access|complete|perform|provide|assist|help|proceed)|unable to (?:review|analy[sz]e)|I must (?:decline|refuse))\b/i.test(text)) return { completed: false, reason: 'review refused' };
|
||||
if (gate === 'spec') {
|
||||
const scores = [...text.matchAll(/^SCORE:[\t ]*(10|[0-9])[\t ]*\r?$/gm)];
|
||||
const ambiguities = [...text.matchAll(/^AMBIGUITIES:[\t ]*(\S[^\r\n]*)\r?$/gm)];
|
||||
if (scores.length !== 1 || ambiguities.length !== 1) return { completed: false, reason: 'missing or invalid SCORE/AMBIGUITIES markers' };
|
||||
const score = Number(scores[0][1]);
|
||||
return { completed: true, score, gate: score >= 7 ? 'pass' : 'fail' };
|
||||
}
|
||||
if (gate === 'structured') {
|
||||
const plain = plainReview(text);
|
||||
const severity = /\[P[0-3]\]|^P[0-3]:/m.test(plain);
|
||||
const clear = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs|issues|findings|problems)\b|\b(?:did not|didn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i.test(text);
|
||||
if (!severity && !clear) return { completed: false, reason: 'missing severity or explicit no-findings conclusion' };
|
||||
return { completed: true, gate: /\[P1\]|^P1:/m.test(plain) ? 'fail' : 'pass' };
|
||||
}
|
||||
// Formatting the requested marker in bold, inline code, or a list does not
|
||||
// invalidate a completed review. Preserve the explicit action + reason gate.
|
||||
const plain = plainReview(text);
|
||||
if (!/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) return { completed: false, reason: 'missing review completion recommendation' };
|
||||
return { completed: true };
|
||||
/**
|
||||
* Review-specific completion evidence, separate from provider transport success.
|
||||
*
|
||||
* classifyOutsideReview() is the INV-1 contract: it takes the review text, the
|
||||
* provider's stderr and exit status (and, for `codex exec --json`, the event
|
||||
* stream) and returns three separate answers: did the review execute, what is
|
||||
* the highest finding severity, and the verdict. validateOutsideReview() is the
|
||||
* older text-only shape, kept for direct importers and stale rendered skills.
|
||||
*/
|
||||
import { GATE_OUTCOMES, gateOutcomeLine, type GateReason } from './gate-outcomes';
|
||||
|
||||
export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution';
|
||||
export type OutsideVerdict = 'clean' | 'findings' | 'unverified' | 'unavailable';
|
||||
export type Severity = 'P0' | 'P1' | 'P2' | 'P3';
|
||||
|
||||
export interface OutsideReviewInput {
|
||||
text: string;
|
||||
gate: OutsideGate;
|
||||
stderr?: string;
|
||||
exit?: number;
|
||||
/** JSONL from `codex exec --json`: command events are positive execution evidence. */
|
||||
events?: string;
|
||||
}
|
||||
|
||||
export interface OutsideReviewClassification {
|
||||
execution: { state: 'ran' } | { state: 'unavailable'; reason: GateReason; detail?: string };
|
||||
findings: { highest: Severity | null };
|
||||
verdict: OutsideVerdict;
|
||||
/** Set for every verdict except clean/findings. */
|
||||
reason?: GateReason;
|
||||
detail?: string;
|
||||
score?: number;
|
||||
}
|
||||
|
||||
const REFUSAL = /\b(?:(?:I (?:cannot|can't|won't|will not|am unable to)|I'm unable to)\s+(?:review|analy[sz]e|evaluate|assess|inspect|access|complete|perform|provide|assist|help|proceed)|unable to (?:review|analy[sz]e)|I must (?:decline|refuse))\b/i;
|
||||
|
||||
/**
|
||||
* Sandbox setup failures as Codex and bubblewrap print them on stderr (fixtures:
|
||||
* test/fixtures/codex-sandbox/). Read from stderr and failed command output
|
||||
* only: a review that merely discusses bwrap or namespaces must still pass, and
|
||||
* a healthy run warns "could not find bubblewrap on PATH" while working.
|
||||
*/
|
||||
const SANDBOX_FAILURE = /^\s*bwrap: \S.*$|^.*\bbubblewrap is unavailable\b.*$|^.*\blandlock\b.{0,60}\b(?:fail\w*|error|not supported|unsupported)\b.*$|^.*\bseccomp\b.{0,60}\b(?:fail\w*|error)\b.*$|^.*\buser namespaces?\b.{0,80}\b(?:not (?:allowed|permitted|supported)|denied|disabled)\b.*$/im;
|
||||
/** Exact phrases a reviewer uses when it could not execute; the last fallback, used only without positive evidence. */
|
||||
const EXECUTION_FAILURE_PHRASE = /\b(?:commands? (?:could not|couldn't|cannot|can't) (?:be )?run|(?:could not|couldn't|was unable to|am unable to|unable to) (?:run (?:any )?(?:shell )?commands|execute (?:any )?commands|inspect the diff|read the diff|access the diff)|the diff could not be (?:read|inspected|accessed)|every (?:shell )?(?:command|invocation) failed)\b/i;
|
||||
/** `codex review` transcript on stderr: a command that ran prints " succeeded in Nms:". */
|
||||
const TRANSCRIPT_SUCCESS = /^\s*succeeded in \d+(?:\.\d+)?m?s:?\s*$/m;
|
||||
|
||||
function commandEvidence(events: string): { attempted: number; succeeded: number; failedOutput: string } {
|
||||
let attempted = 0, succeeded = 0, failedOutput = '';
|
||||
for (const line of events.split(/\r?\n/)) {
|
||||
if (!line.trim().startsWith('{')) continue;
|
||||
let event: any;
|
||||
try { event = JSON.parse(line); } catch { continue; }
|
||||
const item = event?.item;
|
||||
if (event?.type !== 'item.completed' || item?.type !== 'command_execution') continue;
|
||||
attempted++;
|
||||
if (item.status === 'completed' && item.exit_code === 0) succeeded++;
|
||||
else failedOutput += `${typeof item.aggregated_output === 'string' ? item.aggregated_output : ''}\n`;
|
||||
}
|
||||
return { attempted, succeeded, failedOutput };
|
||||
}
|
||||
|
||||
function stderrHead(stderr: string): string | undefined {
|
||||
return stderr.split(/\r?\n/).map(line => line.trim()).find(Boolean)?.slice(0, 240);
|
||||
}
|
||||
|
||||
function execution(input: OutsideReviewInput): OutsideReviewClassification['execution'] {
|
||||
const stderr = input.stderr ?? '';
|
||||
const exit = input.exit ?? 0;
|
||||
const sandbox = (output: string) => output.match(SANDBOX_FAILURE)?.[0].trim().slice(0, 240);
|
||||
if (exit !== 0) {
|
||||
const detail = sandbox(stderr);
|
||||
if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail };
|
||||
const head = stderrHead(stderr);
|
||||
return { state: 'unavailable', reason: exit === 124 ? 'timeout' : 'execution_failed', detail: head ? `exit ${exit}: ${head}` : `exit ${exit}` };
|
||||
}
|
||||
const commands = commandEvidence(input.events ?? '');
|
||||
const executed = commands.succeeded > 0 || TRANSCRIPT_SUCCESS.test(stderr);
|
||||
if (!executed) {
|
||||
const detail = sandbox(stderr) ?? sandbox(commands.failedOutput);
|
||||
if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail };
|
||||
if (commands.attempted > 0) return { state: 'unavailable', reason: 'commands_failed', detail: `all ${commands.attempted} commands failed` };
|
||||
}
|
||||
if (!input.text.trim()) return { state: 'unavailable', reason: 'empty_response' };
|
||||
const phrase = executed ? undefined : input.text.match(EXECUTION_FAILURE_PHRASE)?.[0];
|
||||
if (phrase) return { state: 'unavailable', reason: 'commands_failed', detail: `the review says "${phrase}"` };
|
||||
if (REFUSAL.test(input.text)) return { state: 'unavailable', reason: 'review_refused' };
|
||||
return { state: 'ran' };
|
||||
}
|
||||
|
||||
/** Formatting the requested marker in bold, inline code, or a list does not invalidate a completed review. */
|
||||
function plainReview(text: string): string {
|
||||
return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n');
|
||||
}
|
||||
|
||||
export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification {
|
||||
const plain = plainReview(input.text);
|
||||
const levels = [...plain.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity);
|
||||
const findings = { highest: levels.length ? levels.sort()[0]! : null };
|
||||
const ran = execution(input);
|
||||
if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail };
|
||||
const blocking = findings.highest === 'P0' || findings.highest === 'P1';
|
||||
const result = (verdict: OutsideVerdict, reason?: GateReason, detail?: string): OutsideReviewClassification =>
|
||||
({ execution: ran, findings, verdict, ...(reason ? { reason } : {}), ...(detail ? { detail } : {}) });
|
||||
if (input.gate === 'spec') {
|
||||
const scores = [...input.text.matchAll(/^SCORE:[\t ]*(10|[0-9])[\t ]*\r?$/gm)];
|
||||
const ambiguities = [...input.text.matchAll(/^AMBIGUITIES:[\t ]*(\S[^\r\n]*)\r?$/gm)];
|
||||
if (scores.length !== 1 || ambiguities.length !== 1) return result('unavailable', 'missing_markers', 'missing or invalid SCORE/AMBIGUITIES markers');
|
||||
const score = Number(scores[0]![1]);
|
||||
return { ...result(score >= 7 ? 'clean' : 'findings'), score };
|
||||
}
|
||||
if (input.gate === 'structured') {
|
||||
const clear = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs|issues|findings|problems)\b|\b(?:did not|didn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i.test(input.text);
|
||||
if (!findings.highest && !clear) return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
|
||||
return result(blocking ? 'findings' : 'clean');
|
||||
}
|
||||
if (input.gate === 'review' && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) {
|
||||
return result('unavailable', 'missing_markers', 'missing review completion recommendation');
|
||||
}
|
||||
return result(blocking ? 'findings' : 'clean');
|
||||
}
|
||||
|
||||
const LEGACY_REASONS: Partial<Record<GateReason, string>> = {
|
||||
empty_response: 'empty response',
|
||||
review_refused: 'review refused',
|
||||
};
|
||||
|
||||
/** Text-only compatibility shape for direct importers (claude-code/SKILL.md.tmpl) and the two-argument CLI. */
|
||||
export function validateOutsideReview(text: string, gate: OutsideGate): { completed: boolean; reason?: string; score?: number; gate?: 'pass' | 'fail' } {
|
||||
const checked = classifyOutsideReview({ text, gate });
|
||||
if (checked.verdict === 'unavailable' || checked.verdict === 'unverified') {
|
||||
const reason = LEGACY_REASONS[checked.reason!] ?? checked.detail ?? GATE_OUTCOMES[checked.reason!].summary;
|
||||
return { completed: false, reason };
|
||||
}
|
||||
if (gate === 'spec') return { completed: true, score: checked.score, gate: checked.verdict === 'clean' ? 'pass' : 'fail' };
|
||||
if (gate === 'structured') return { completed: true, gate: checked.verdict === 'findings' ? 'fail' : 'pass' };
|
||||
return { completed: true };
|
||||
}
|
||||
|
||||
const VERDICT_EXIT: Record<OutsideVerdict, number> = { clean: 0, findings: 3, unverified: 4, unavailable: 1 };
|
||||
const GATES = ['review', 'structured', 'spec', 'execution'];
|
||||
const USAGE = `Usage: outside-review-result.ts <gate> <response-file>
|
||||
outside-review-result.ts --verdict [--stderr <file>] [--exit <code>] [--events <file>] [--label <name>] <gate> <response-file>
|
||||
Gates: ${GATES.join('|')}. Two-argument form exits 0 completed, 1 unavailable, 2 usage.
|
||||
Verdict form prints VERDICT: clean|findings|unverified|unavailable and exits 0, 3, 4 or 1.`;
|
||||
|
||||
async function readOptional(file: string | undefined): Promise<string> {
|
||||
if (!file) return '';
|
||||
try { return await Bun.file(file).text(); } catch { return ''; }
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const [gate, path] = process.argv.slice(2);
|
||||
if (!['review', 'structured', 'spec'].includes(gate) || !path) { console.error('Usage: outside-review-result.ts review|structured|spec <response-file>'); process.exit(2); }
|
||||
try {
|
||||
const result = validateOutsideReview(await Bun.file(path).text(), gate as OutsideGate);
|
||||
const args = process.argv.slice(2);
|
||||
const flags: Record<string, string> = {};
|
||||
const positional: string[] = [];
|
||||
let verdictMode = false;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === '--verdict') verdictMode = true;
|
||||
else if (['--stderr', '--exit', '--events', '--label'].includes(arg) && i + 1 < args.length) { flags[arg] = args[++i]!; verdictMode ||= arg !== '--label'; }
|
||||
else if (arg.startsWith('--')) { console.error(USAGE); process.exit(2); }
|
||||
else positional.push(arg);
|
||||
}
|
||||
const [gate, path] = positional;
|
||||
const exit = flags['--exit'] === undefined ? 0 : Number(flags['--exit']);
|
||||
if (!gate || !GATES.includes(gate) || !path || positional.length !== 2 || !Number.isInteger(exit)
|
||||
|| (!verdictMode && gate === 'execution')) { console.error(USAGE); process.exit(2); }
|
||||
let text: string;
|
||||
try { text = await Bun.file(path).text(); } catch (error) {
|
||||
if (!verdictMode) { console.error(`Outside review unavailable: ${error}`); process.exit(1); }
|
||||
text = '';
|
||||
}
|
||||
if (!verdictMode) {
|
||||
const result = validateOutsideReview(text, gate as OutsideGate);
|
||||
if (!result.completed) { console.error(`Outside review unavailable: ${result.reason}; missing coverage.`); process.exit(1); }
|
||||
} catch (error) { console.error(`Outside review unavailable: ${error}`); process.exit(1); }
|
||||
process.exit(0);
|
||||
}
|
||||
const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit,
|
||||
stderr: await readOptional(flags['--stderr']), events: await readOptional(flags['--events']) });
|
||||
console.log(`VERDICT: ${checked.verdict}`);
|
||||
console.log(`FINDINGS: ${checked.findings.highest ?? 'none'}`);
|
||||
if (checked.reason) {
|
||||
console.log(`REASON: ${checked.reason}`);
|
||||
console.error(gateOutcomeLine(flags['--label'] ?? 'Outside review', checked.reason, checked.detail));
|
||||
}
|
||||
process.exit(VERDICT_EXIT[checked.verdict]);
|
||||
}
|
||||
@@ -110,7 +110,8 @@ On any error: continue — ${feature} is informational, not a gate.`;
|
||||
* 5. sets ONE canonical mode var and echoes `CODEX_MODE: <mode>` so the agent
|
||||
* gates later blocks on the echoed value.
|
||||
*
|
||||
* Mode values: `disabled` (config off) | `not_installed` | `not_authed` | `ready`.
|
||||
* Mode values: `disabled` (config off) | `not_installed` | `not_authed` |
|
||||
* `broken_install` | `sandbox_unavailable` | `model_unusable` | `unverified` | `ready`.
|
||||
* The path is host-rewritten at gen-skill-docs time (pathRewrites), so the
|
||||
* literal `~/.claude/skills/gstack` is correct here and becomes `$GSTACK_ROOT`
|
||||
* etc. for non-Claude hosts.
|
||||
@@ -144,14 +145,19 @@ elif ! command -v codex >/dev/null 2>&1; then
|
||||
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
|
||||
${m}="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
|
||||
else
|
||||
# Capture the probe's code: 2 means the CLI cannot execute at all, which is a
|
||||
# different problem (and a different fix) from a model the account can't use.
|
||||
_gstack_codex_model_probe; _CODEX_MP=$?${opts.nativeReview ? `
|
||||
# The free sandbox check runs before the paid model probe. Probe code 2 means
|
||||
# the CLI cannot execute at all, a different fix from an unusable model.
|
||||
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }${opts.nativeReview ? `
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe review; _CODEX_MP=$?; }` : ''}
|
||||
if [ "$_CODEX_MP" -eq 2 ]; then
|
||||
if [ "$_CODEX_MP" -eq 3 ]; then
|
||||
${m}="sandbox_unavailable"
|
||||
elif [ "$_CODEX_MP" -eq 2 ]; then
|
||||
${m}="broken_install"
|
||||
elif [ "$_CODEX_MP" -ne 0 ]; then
|
||||
${m}="model_unusable"
|
||||
elif [ "\${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
|
||||
${m}="unverified"
|
||||
else
|
||||
${m}="ready"; _gstack_codex_version_check 2>/dev/null || true
|
||||
fi
|
||||
@@ -165,8 +171,9 @@ Branch on the echoed \`CODEX_MODE\`:
|
||||
- **\`under_codex\`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider.
|
||||
- **\`not_authed\`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run \`codex login\` or set \`$CODEX_API_KEY\`." ${nativeRoute}
|
||||
- **\`broken_install\`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: \`npm install -g @openai/codex\`." Relay the probe's HINT lines. ${nativeRoute}
|
||||
- **\`model_unusable\`** — the selected model (see \`CODEX_MODEL:\`) is invalid or unavailable to the account (#2477: HTTP 400 on every call). Relay the probe's HINT lines and the fix (\`GSTACK_CODEX_MODEL=<supported-model>\` or config.toml \`model\`); never substitute a model. ${nativeRoute} The ~10s round trip is cached for 1h; timeouts fail open to \`ready\`.
|
||||
- **\`ready\`** — run the Codex pass below.`;
|
||||
- **\`model_unusable\`** — the selected model (see \`CODEX_MODEL:\`) is invalid or unavailable to the account (#2477: HTTP 400 on every call). Relay the probe's HINT lines and the fix (\`GSTACK_CODEX_MODEL=<supported-model>\` or config.toml \`model\`); never substitute a model. ${nativeRoute} The ~10s round trip is cached for 1h.
|
||||
- **\`sandbox_unavailable\`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. ${nativeRoute}
|
||||
- **\`ready\`** or **\`unverified\`** — run the Codex pass below. \`unverified\` means the model check timed out; say so, and let the pass's own verdict decide.`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -141,7 +141,7 @@ If the subagent fails or times out, record native coverage as incomplete. Contin
|
||||
}
|
||||
|
||||
function adversarialOutsideChallenge(ctx: TemplateContext, isShip: boolean): string {
|
||||
return `### ${outsideVoiceFor(ctx).label} adversarial challenge (runs whenever \`CODEX_MODE: ready\`)
|
||||
return `### ${outsideVoiceFor(ctx).label} adversarial challenge (runs whenever \`CODEX_MODE\` is \`ready\` or \`unverified\`)
|
||||
|
||||
If \`CODEX_MODE\` is \`ready\`:
|
||||
|
||||
@@ -151,7 +151,7 @@ Outside prompt (supply repository context from the parent):
|
||||
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 540000, nativeAlreadyRequired: true, diffCommand: 'DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff "$DIFF_BASE"' })}
|
||||
|
||||
Present the full output verbatim. ${isShip ? 'An unavailable outside challenge does not block shipping by itself; supported findings still enter Step 11, and the structured P1 and non-convergence gates still apply.' : 'This outside challenge is informational; supported findings still enter Step 5 Fix-First, whose approval and convergence gates apply.'}
|
||||
Present the full output verbatim. ${isShip ? 'An unavailable outside challenge does not block shipping by itself; supported findings still enter Step 11, and the structured P0/P1 and non-convergence gates still apply.' : 'This outside challenge is informational; supported findings still enter Step 5 Fix-First, whose approval and convergence gates apply.'}
|
||||
|
||||
**Error handling:** Only this optional outside adversarial pass is non-blocking; native completion and structured-review decisions still apply.
|
||||
${outsideVoiceFailurePolicy(ctx, { timeoutMinutes: 9, onTimeout: 'missing-coverage', stderrOnEmpty: true, fallback: 'none', escape: 1 })}
|
||||
@@ -168,14 +168,14 @@ function adversarialStructuredReview(ctx: TemplateContext, isShip: boolean): str
|
||||
|
||||
If \`CODEX_MODE\` is \`ready\` and either \`DIFF_TOTAL >= 200\` or the user requested the override above:
|
||||
|
||||
Prepare a structured review prompt requesting severity-tagged findings ([P1], [P2], [P3]) or an explicit NO_FINDINGS conclusion. Preserve the base-branch scope including committed changes and working-tree changes.
|
||||
Prepare a structured review prompt requesting severity-tagged findings ([P0]-[P3]) or an explicit NO_FINDINGS conclusion. Preserve the base-branch scope including committed changes and working-tree changes.
|
||||
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 540000, nativeAlreadyRequired: true, structuredBase: '<base>', gate: 'structured', diffCommand: 'DIFF_BASE=$(git merge-base <base> HEAD) && git diff "$DIFF_BASE"' })}
|
||||
|
||||
${outsideVoiceFor(ctx).id === 'codex' ? 'The Codex backend uses `codex review --base` without a positional prompt: those arguments are mutually exclusive. Never drop --base to resolve an argv error; prompt-only review changes the diff scope.' : 'The Claude Code backend receives the parent-captured base diff, including committed and working-tree changes, because review mode cannot execute git.'}
|
||||
|
||||
Present output under \`${outsideVoiceFor(ctx).label.toUpperCase()} SAYS (code review):\` inside a \`tool-output\` fence.
|
||||
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P1 findings (\`[P1]\` or native \`P1:\` labels) → GATE: FAIL. Completed without P1 → GATE: PASS. Refusal, failure, or missing markers → GATE: MISSING COVERAGE; preserve the existing user decision flow.
|
||||
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P0/P1 findings (\`[P0]\`/\`[P1]\` or native \`P0:\`/\`P1:\` labels; \`VERDICT: findings\`) → GATE: FAIL. Completed without P0/P1 → GATE: PASS. Refusal, failure, missing markers or \`OUTSIDE_STATUS: unverified\` → GATE: MISSING COVERAGE; preserve the existing user decision flow.
|
||||
|
||||
If GATE is FAIL, use AskUserQuestion:
|
||||
\`\`\`
|
||||
@@ -255,7 +255,7 @@ ${outsideVoicePreflight(ctx, { disabledBehavior: 'codex-only', nativeReview: tru
|
||||
\`ready\` runs them; every other mode skips them with the printed reason.
|
||||
The ${outsideVoiceFor(ctx).nativeLabel} adversarial subagent always runs.
|
||||
|
||||
**User override:** If the user explicitly requested "full review", "structured review", or "P1 gate", also run the ${outsideVoiceFor(ctx).label} structured review regardless of diff size (still requires \`CODEX_MODE: ready\`).
|
||||
**User override:** If the user explicitly requested "full review", "structured review", or "P1 gate", also run the ${outsideVoiceFor(ctx).label} structured review regardless of diff size (still requires \`CODEX_MODE: ready\` (or \`unverified\`)).
|
||||
|
||||
---
|
||||
|
||||
@@ -406,7 +406,7 @@ THE PLAN:
|
||||
}
|
||||
|
||||
function codexPlanReviewRun(ctx: TemplateContext, ceo: boolean, needsApprovalReadiness: boolean): string {
|
||||
return `**If \`CODEX_MODE: ready\` — run ${outsideVoiceFor(ctx).label}:**
|
||||
return `**If \`CODEX_MODE: ready\` (or \`unverified\`) — run ${outsideVoiceFor(ctx).label}:**
|
||||
|
||||
${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) ? `Run this block only for \`ready\`, in the one foreground Bash call described below.
|
||||
Its opening harness guard rechecks the fresh shell: exit 78 uses the same Native
|
||||
@@ -717,7 +717,7 @@ numbers, and CHANGELOG entries that over- or under-sell what shipped. Be terse.
|
||||
|
||||
THE DOCS AND DIFF: <include current contents of each touched document, with its path, plus affected source context; the parent appends the release diff below>"
|
||||
|
||||
**If \`CODEX_MODE: ready\` — run ${outsideVoiceFor(ctx).label}:**
|
||||
**If \`CODEX_MODE: ready\` (or \`unverified\`) — run ${outsideVoiceFor(ctx).label}:**
|
||||
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, diffCommand: 'DOC_DIFF_BASE=$(git merge-base origin/<base> HEAD 2>/dev/null || git merge-base <base> HEAD) && git diff "$DOC_DIFF_BASE" HEAD' })}
|
||||
|
||||
|
||||
@@ -128,13 +128,15 @@ export function outsideVoiceCommand(ctx: TemplateContext, opts: OutsideCommandOp
|
||||
const root = toShellPath(ctx.paths.skillRoot);
|
||||
const prompt = sh(opts.promptFile ?? '<prepared-prompt-file>');
|
||||
const codex = opts.structuredBase
|
||||
? `codex review --base ${sh(opts.structuredBase)} ${CODEX_REVIEW_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} < /dev/null`
|
||||
: `codex exec "$_OUTSIDE_PROMPT" -C "$_REPO_ROOT" -s read-only ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} < /dev/null`;
|
||||
? `codex review --base ${sh(opts.structuredBase)} -c "sandbox_mode=\\"\${_GSTACK_CODEX_SANDBOX:?}\\"" ${CODEX_REVIEW_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} < /dev/null`
|
||||
: `codex exec - -C "$_REPO_ROOT" -s "\${_GSTACK_CODEX_SANDBOX:?}" ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT"`;
|
||||
const invocation = v.id === 'codex'
|
||||
? `source "${bin}/gstack-codex-probe" && _gstack_codex_select_model ${opts.structuredBase ? 'review' : 'exec'} || exit 1
|
||||
${opts.structuredBase ? '' : '_OUTSIDE_PROMPT=$(cat "$_OUTSIDE_INPUT") || exit 1\n'}_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper ${Math.ceil(opts.timeoutMs / 1000)} ${codex} >"$_OUTSIDE_TMP/text" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
cat "$_OUTSIDE_TMP/text" || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; }`
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper ${Math.ceil(opts.timeoutMs / 1000)} ${codex} >"$_OUTSIDE_TMP/${opts.structuredBase ? 'text' : 'events'}" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
${opts.structuredBase ? 'cat "$_OUTSIDE_TMP/text"' : 'cat "$_OUTSIDE_TMP/text" 2>/dev/null || tail -n 20 "$_OUTSIDE_TMP/events"'}`
|
||||
: `_OUTSIDE_EXIT=0
|
||||
"${bin}/gstack-claude-code" --cwd "$_REPO_ROOT" --access ${opts.access ?? 'none'} --timeout-ms ${opts.timeoutMs} <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/result.json" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
# Preserve session/usage/modelUsage from this JSON; multiple models have no invented primary.
|
||||
@@ -158,12 +160,13 @@ ${v.id === 'codex' && ctx.skillName === 'autoplan' ? `if [ "$_OUTSIDE_EXIT" -eq
|
||||
_gstack_codex_log_hang "autoplan" "0" || true
|
||||
fi` : ''}
|
||||
cat "$_OUTSIDE_TMP/stderr" >&2 || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; }
|
||||
if [ "$_OUTSIDE_EXIT" -ne 0 ]; then
|
||||
echo '${v.label} outside review unavailable: execution failed; missing coverage. Check the provider diagnosis above.' >&2
|
||||
exit "$_OUTSIDE_EXIT"
|
||||
fi
|
||||
bun "${root}/lib/outside-review-result.ts" ${opts.gate ?? 'review'} "$_OUTSIDE_TMP/text" || exit 1
|
||||
${v.id === 'claude-code' ? 'cat "$_OUTSIDE_TMP/text" || exit 1' : ''}
|
||||
_OUTSIDE_RC=0
|
||||
bun "${root}/lib/outside-review-result.ts" --label '${v.label} outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" ${v.id === 'codex' && !opts.structuredBase ? '--events "$_OUTSIDE_TMP/events" ' : ''}${opts.gate ?? 'review'} "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$?
|
||||
${v.id === 'claude-code' ? '[ "$_OUTSIDE_RC" -eq 1 ] || cat "$_OUTSIDE_TMP/text" || exit 1\n' : ''}case "$_OUTSIDE_RC" in
|
||||
0|3) ;;
|
||||
4) echo 'OUTSIDE_STATUS: unverified provider=${v.id} host=${ctx.host}'; exit 4 ;;
|
||||
*) [ "$_OUTSIDE_EXIT" -ne 0 ] && exit "$_OUTSIDE_EXIT"; exit 1 ;;
|
||||
esac
|
||||
echo 'OUTSIDE_STATUS: completed provider=${v.id} host=${ctx.host}'`;
|
||||
}
|
||||
|
||||
@@ -197,7 +200,7 @@ export function outsideVoiceInvocation(ctx: TemplateContext, requested: OutsideC
|
||||
${outsideVoiceCommand(ctx, opts)}
|
||||
\`\`\`
|
||||
|
||||
Use Bash \`timeout: ${gateMs}\`; show the full response in a \`tool-output\` fence. Require successful execution and valid markers. Refusal, empty/malformed output, missing ${planRecommendation ? 'Recommendation: <action> because <reason>' : opts.purpose === 'design-direction' ? 'Recommendation' : 'score/severity/completion'} markers, timeout or CLI failure means \`outside_status: unavailable\`. ${opts.purpose === 'design-direction' ? 'Continue completed proposals; native completion does not count as outside coverage.' : opts.nativeAlreadyRequired ? 'Retain the required native pass without duplicating it; it cannot complete outside coverage.' : "Use the caller's fallback; missing coverage is never clean/PASS."} ${nativeStructured ? 'Scratch cleanup is automatic.' : 'After either outcome, delete only your private prompt; scratch cleanup is automatic.'}`;
|
||||
Use Bash \`timeout: ${gateMs}\`; show the full response in a \`tool-output\` fence. Require successful execution and valid markers. Refusal, empty/malformed output, missing ${planRecommendation ? 'Recommendation: <action> because <reason>' : opts.purpose === 'design-direction' ? 'Recommendation' : 'score/severity/completion'} markers, timeout or CLI failure means \`outside_status: unavailable\`. P0/P1 findings block like native ones; \`OUTSIDE_STATUS: unverified\` is missing coverage. ${opts.purpose === 'design-direction' ? 'Continue completed proposals; native completion does not count as outside coverage.' : opts.nativeAlreadyRequired ? 'Retain the required native pass without duplicating it; it cannot complete outside coverage.' : "Use the caller's fallback; missing coverage is never clean/PASS."} ${nativeStructured ? 'Scratch cleanup is automatic.' : 'After either outcome, delete only your private prompt; scratch cleanup is automatic.'}`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -240,7 +243,7 @@ export function outsideVoiceFailurePolicy(ctx: TemplateContext, opts: OutsideVoi
|
||||
|
||||
export function outsideVoiceProvenance(ctx: TemplateContext, phase: string): string {
|
||||
const v = outsideVoiceFor(ctx);
|
||||
return `Retain the historical review-log skill ID; add \`"host":"${ctx.host}","outside_provider":"${v.id}","outside_status":"completed|unavailable|disabled|skipped","phase":"${phase}"\`. Record differing attempt outcomes separately. \`source:"${v.id}"\` requires completed CLI output; native uses \`source:"in-host"\` (historical \`source:"claude"\`: native Claude). Availability/native fallback is not outside completion. Preserve all reported modelUsage; unknown model identity stays unknown.`;
|
||||
return `Retain the historical review-log skill ID; add \`"host":"${ctx.host}","outside_provider":"${v.id}","outside_status":"completed|unavailable|disabled|skipped","phase":"${phase}"\`. Record differing attempt outcomes separately. \`source:"${v.id}"\` requires completed CLI output; native uses \`source:"in-host"\` (historical \`source:"claude"\`: native Claude). Availability/native fallback is not outside completion. Preserve all reported modelUsage; unknown model identity stays unknown.${v.id === 'codex' ? ' Under `GSTACK_CODEX_NO_SANDBOX=1` add `"codex_sandbox":"danger-full-access"`.' : ''}`;
|
||||
}
|
||||
|
||||
export function generateOutsideVoiceRouting(ctx: TemplateContext): string {
|
||||
|
||||
@@ -29,17 +29,20 @@ function fixture(plan?:string){
|
||||
return {dir,method,snapshot,file,events,options,read:(rows=events)=>autoplanDualVoiceEvidence(rows,options)};
|
||||
}
|
||||
test('actual6bd source Read, unused probe branch and0Hspec Agent earn zero phase voice credit',()=>{
|
||||
const f=fixture(),actual=f.read(clone(captured.events));
|
||||
const f=fixture();
|
||||
// Judge the capture against the probe block it executed (B1 later added the sandbox preflight).
|
||||
f.options.commands={...f.options.commands,probe:captured.sourceBoundB176.commandContract.probe};
|
||||
const actual=f.read(clone(captured.events));
|
||||
expect(actual).toMatchObject({claudeVoiceFired:false,codexVoiceFired:false,codexUnavailable:false,reviewDispatched:false,probeMode:'ready'});
|
||||
});
|
||||
test('actual snapshot producer and complete parent request/ACK pairs establish both voices',()=>{
|
||||
const f=fixture();expect(f.read()).toMatchObject({claudeVoiceFired:true,codexVoiceFired:true,codexUnavailable:false,reviewDispatched:true,nativeToolUseId:'native',outsideToolUseId:'outside'});
|
||||
});
|
||||
test.each(['not_installed','not_authed','broken_install','model_unusable'])('actual final probe result supports unavailable fallback: %s',mode=>{
|
||||
test.each(['not_installed','not_authed','broken_install','sandbox_unavailable','model_unusable'])('actual final probe result supports unavailable fallback: %s',mode=>{
|
||||
const f=fixture();f.events.splice(4);f.events[1]=ack('probe','CODEX_MODE: '+mode);
|
||||
expect(f.read()).toMatchObject({claudeVoiceFired:true,codexVoiceFired:false,codexUnavailable:true});
|
||||
});
|
||||
test.each(['ready','disabled','under_codex','unknown'])('probe outcome supplies no unavailable credit: %s',mode=>{
|
||||
test.each(['ready','unverified','disabled','under_codex','unknown'])('probe outcome supplies no unavailable credit: %s',mode=>{
|
||||
const f=fixture();f.events.splice(4);f.events[1]=ack('probe','CODEX_MODE: '+mode);expect(f.read().codexUnavailable).toBe(false);
|
||||
});
|
||||
test.each(['missing','error','foreign','child','unowned','method','phase','math','no-launch','wrong-input','mutable'])('native dispatch rejects %s evidence',kind=>{
|
||||
@@ -169,6 +172,8 @@ test.each([
|
||||
'Outside review unavailable: empty response; missing coverage.',
|
||||
'Outside review unavailable: review refused; missing coverage.',
|
||||
'Outside review unavailable: missing review completion recommendation; missing coverage.',
|
||||
'Codex outside review unavailable: the reviewer process failed (exit 1: error: 401 Unauthorized). No review ran; this is missing coverage, not a pass. Fix: read the provider diagnosis above (auth, model, network), repair it, then re-run the review.',
|
||||
'Codex outside review unavailable: the response lacks the markers this gate requires (missing review completion recommendation). No review ran; this is missing coverage, not a pass. Fix: re-run the review; a response without its required markers never counts as a pass.',
|
||||
])('actual owned post-execution error proves attempted outside voice, not completion: %s',diagnostic=>{
|
||||
const f=fixture();f.events[7]=ack('outside','Exit code 1\n'+diagnostic,true);
|
||||
expect(f.read()).toMatchObject({claudeVoiceFired:true,codexAttempted:true,codexVoiceFired:false,codexUnavailable:true,failedOutsideToolUseId:'outside'});
|
||||
@@ -259,8 +264,8 @@ test.each(['assignment-only','set-config','other-config','foreign-reader','or-co
|
||||
if(kind==='before-cd')input.command=guards+'\ncd '+f.dir+'\n'+f.options.commands.outside.replace("'<prepared-prompt-file>'","'"+f.file+"'");
|
||||
if(kind==='changed-harness')input.command=input.command.replace('exit 78','exit 0');
|
||||
if(kind==='changed-timeout')input.command=input.command.replace('_gstack_codex_timeout_wrapper 540','_gstack_codex_timeout_wrapper 1');
|
||||
if(kind==='changed-sandbox')input.command=input.command.replace('-s read-only','-s danger-full-access');
|
||||
if(kind==='changed-prompt')input.command=input.command.replace('codex exec "$_OUTSIDE_PROMPT"','codex exec "Different plan"');
|
||||
if(kind==='changed-sandbox')input.command=input.command.replace('-s "${_GSTACK_CODEX_SANDBOX:?}"','-s danger-full-access');
|
||||
if(kind==='changed-prompt')input.command=input.command.replace('codex exec - ','codex exec "Different plan" ');
|
||||
if(kind==='skipped-validator')input.command=input.command.replace(/^bun .*outside-review-result.*\n/m,'');
|
||||
if(kind==='suffix')input.command+='\ntrue';
|
||||
expect(f.read().codexVoiceFired,kind).toBe(false);
|
||||
|
||||
@@ -51,6 +51,44 @@ function tempHome(): string {
|
||||
}
|
||||
|
||||
describe('gstack-codex-probe: auth probe', () => {
|
||||
// #2192: custom OpenAI-compatible providers name their credential variable
|
||||
// in config.toml [model_providers.<id>] env_key; a set variable is auth.
|
||||
const withConfig = (toml: string, env: Record<string, string | undefined> = {}) => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
fs.mkdirSync(path.join(home, '.codex'), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, '.codex', 'config.toml'), toml);
|
||||
return runProbe({ snippet: '_gstack_codex_auth_probe', env, home });
|
||||
} finally { fs.rmSync(home, { recursive: true, force: true }); }
|
||||
};
|
||||
const MIMO = '[model_providers.mimo]\nenv_key = "MIMO_API_KEY"\n';
|
||||
|
||||
test('#2192: a provider env_key whose variable is set → AUTH_OK', () => {
|
||||
const r = withConfig(MIMO, { MIMO_API_KEY: 'tp-test-key' });
|
||||
expect([r.stdout.trim(), r.status]).toEqual(['AUTH_OK', 0]);
|
||||
const several = withConfig('[model_providers.a]\nenv_key = "MISSING_A"\n\n[model_providers.b]\nenv_key = \'MIMO_API_KEY\' # inline comment\n', { MIMO_API_KEY: 'k' });
|
||||
expect(several.stdout.trim()).toBe('AUTH_OK');
|
||||
});
|
||||
|
||||
test('#2192: a provider env_key that is unset or blank → AUTH_FAILED naming the key', () => {
|
||||
for (const value of [undefined, ' \t\n']) {
|
||||
const r = withConfig(MIMO, { MIMO_API_KEY: value });
|
||||
expect([r.stdout.trim(), r.status]).toEqual(['AUTH_FAILED', 1]);
|
||||
expect(r.stderr).toContain('custom provider key (MIMO_API_KEY)');
|
||||
}
|
||||
});
|
||||
|
||||
test('#2192: commented env_key lines and env_key outside [model_providers.*] never count', () => {
|
||||
const toml = '[model_providers.mimo]\n# env_key = "DEPRECATED_KEY"\nenv_key = "MIMO_API_KEY"\n[profiles.work]\nenv_key = "PROFILE_KEY"\n';
|
||||
expect(withConfig(toml, { DEPRECATED_KEY: 'old', PROFILE_KEY: 'p' }).stdout.trim()).toBe('AUTH_FAILED');
|
||||
});
|
||||
|
||||
test('#2192: malformed config.toml or a non-identifier env_key → AUTH_FAILED, no crash', () => {
|
||||
expect(withConfig('\x00\x01 garbage \xff[broken\n=no=\n').stdout.trim()).toBe('AUTH_FAILED');
|
||||
const injected = withConfig('[model_providers.x]\nenv_key = "A;touch /tmp/never"\n');
|
||||
expect([injected.stdout.trim(), injected.status]).toEqual(['AUTH_FAILED', 1]);
|
||||
});
|
||||
|
||||
test('CODEX_API_KEY set → AUTH_OK', () => {
|
||||
const home = tempHome();
|
||||
try {
|
||||
@@ -337,6 +375,25 @@ describe('gstack-codex-probe: timeout wrapper + namespace hygiene', () => {
|
||||
}
|
||||
});
|
||||
|
||||
for (const native of [false, true]) test(`E5: ${native ? 'bash-native watchdog' : 'timeout(1)'} passes the caller's stdin to the command (#1674)`, () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-watchdog-stdin-'));
|
||||
try {
|
||||
for (const tool of native ? ['bash', 'sleep', 'cat', 'printf'] : []) {
|
||||
const resolved = spawnSync('bash', ['-c', `command -v ${tool}`], { timeout: 5000 });
|
||||
const target = resolved.stdout.toString().trim();
|
||||
if (target.startsWith('/')) fs.symlinkSync(target, path.join(dir, tool));
|
||||
}
|
||||
// A redirect onto the wrapper is how callers feed `codex exec -`; a
|
||||
// backgrounded command without job control would otherwise read /dev/null.
|
||||
const input = path.join(dir, 'prompt.txt');
|
||||
fs.writeFileSync(input, 'prompt on stdin');
|
||||
const r = runProbe({ snippet: `_gstack_codex_timeout_wrapper 5 cat < "${input}"; echo " rc=$?"`, env: native ? { PATH: dir } : {} });
|
||||
expect(r.stdout).toBe('prompt on stdin rc=0\n');
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('bash-native watchdog reports its timeout while still retiring after TERM', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-watchdog-race-'));
|
||||
try {
|
||||
@@ -626,13 +683,16 @@ describe('codex skeleton+sections union: review sandbox + fail-closed gate + tim
|
||||
for (const relPath of ['codex tmpl union', 'codex rendered union'] as const) {
|
||||
const read = () => readCodexUnion(relPath === 'codex tmpl union' ? 'tmpl' : 'rendered');
|
||||
|
||||
test(`${relPath}: (a) every scoped codex review invocation pins sandbox_mode="read-only"`, () => {
|
||||
test(`${relPath}: (a) every scoped codex review invocation pins sandbox_mode from the selected sandbox`, () => {
|
||||
const invocations = read()
|
||||
.split('\n')
|
||||
.filter((l) => /_gstack_codex_timeout_wrapper\s+\d+\s+codex\s+review\b/.test(l));
|
||||
expect(invocations.length).toBeGreaterThanOrEqual(1);
|
||||
for (const line of invocations) {
|
||||
expect(line).toContain('sandbox_mode="read-only"');
|
||||
// _gstack_codex_select_model sets read-only (full access only for
|
||||
// GSTACK_CODEX_NO_SANDBOX=1, test/codex-model-probe.test.ts); :? stops
|
||||
// an unselected command instead of inheriting config.toml's default.
|
||||
expect(line).toContain('sandbox_mode=\\"${_GSTACK_CODEX_SANDBOX:?}\\"');
|
||||
// `codex review` has no -s/--sandbox flag (verified 0.147.0) — the
|
||||
// config override is the only lever. `-s read-only` here would fail
|
||||
// at argv parsing, which check (b) would then read as a gate FAIL.
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* B1 static coverage: every rendered fence that runs `codex exec` or
|
||||
* `codex review` as a command routes the result through the shared validator
|
||||
* (lib/outside-review-result.ts) later in the same fence, and takes its
|
||||
* sandbox from `_GSTACK_CODEX_SANDBOX` (read-only unless GSTACK_CODEX_NO_SANDBOX=1);
|
||||
* every `codex exec` reads its prompt on stdin (E5).
|
||||
* Prose mentions of the commands are ignored: only command positions count.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-sites-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
/** `codex exec|review` at the start of a shell command, optionally behind the timeout wrapper. */
|
||||
const COMMAND = /(?:^|[;&|]\s*|\$\(\s*)(?:_gstack_codex_timeout_wrapper\s+\d+\s+)?codex\s+(?:exec|review)\b/;
|
||||
const VALIDATOR = /outside-review-result\.ts\b/;
|
||||
|
||||
interface Site { file: string; line: string; fence: string; after: string }
|
||||
|
||||
function codexSites(markdown: string, file: string): Site[] {
|
||||
const sites: Site[] = [];
|
||||
for (const match of markdown.matchAll(/```bash\n([\s\S]*?)```/g)) {
|
||||
const fence = match[1]!;
|
||||
const lines = fence.split('\n');
|
||||
lines.forEach((line, i) => {
|
||||
const code = line.trim();
|
||||
if (code.startsWith('#') || !COMMAND.test(code)) return;
|
||||
sites.push({ file, line: code, fence, after: lines.slice(i + 1).join('\n') });
|
||||
});
|
||||
}
|
||||
return sites;
|
||||
}
|
||||
|
||||
function render(host: string): string[] {
|
||||
const out = path.join(TMP, host);
|
||||
const r = spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts', '--host', host, '--out-dir', out], { cwd: ROOT, encoding: 'utf8', timeout: 120000 });
|
||||
if (r.status !== 0) throw new Error(`render ${host} failed: ${r.stderr}`);
|
||||
const files: string[] = [];
|
||||
const walk = (dir: string) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(p);
|
||||
else if (entry.name.endsWith('.md')) files.push(p);
|
||||
}
|
||||
};
|
||||
walk(out);
|
||||
return files;
|
||||
}
|
||||
|
||||
describe('every rendered Codex invocation is classified by the shared validator', () => {
|
||||
const rendered = Object.fromEntries(['claude', 'codex'].map(host => [host, render(host).flatMap(f => codexSites(fs.readFileSync(f, 'utf8'), path.relative(TMP, f)))]));
|
||||
|
||||
test('the Claude render has the expected invocation sites', () => {
|
||||
const files = new Set(rendered.claude!.map(s => s.file.replace(/^claude\//, '')));
|
||||
for (const expected of ['codex/sections/review-mode.md', 'codex/sections/challenge-mode.md', 'codex/sections/consult-mode.md',
|
||||
'review/sections/adversarial.md', 'ship/sections/adversarial.md']) expect(files.has(expected)).toBe(true);
|
||||
});
|
||||
|
||||
for (const host of ['claude', 'codex']) {
|
||||
test(`${host}: each codex exec/review command is followed by the validator in its fence`, () => {
|
||||
const missing = rendered[host]!.filter(site => !VALIDATOR.test(site.after)).map(s => `${s.file}: ${s.line}`);
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
test(`${host}: E5 every codex exec reads its prompt on stdin, never argv`, () => {
|
||||
const argv = rendered[host]!.filter(site => /codex\s+exec\b/.test(site.line))
|
||||
.filter(site => !/codex\s+exec\s+(?:resume\s+\S+\s+)?-\s/.test(site.line) || /<\s*\/dev\/null/.test(site.line))
|
||||
.map(s => `${s.file}: ${s.line}`);
|
||||
expect(argv).toEqual([]);
|
||||
});
|
||||
|
||||
test(`${host}: each codex exec/review command takes its sandbox from _GSTACK_CODEX_SANDBOX`, () => {
|
||||
const literal = rendered[host]!.filter(site => !site.line.includes('${_GSTACK_CODEX_SANDBOX:?}')).map(s => `${s.file}: ${s.line}`);
|
||||
expect(literal).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('negative controls: prose is ignored, an unvalidated or literal-sandbox command is flagged', () => {
|
||||
const doc = [
|
||||
'Run `codex review` to review.',
|
||||
'```bash',
|
||||
'# codex exec in a comment',
|
||||
'echo "use codex exec later"',
|
||||
'_gstack_codex_timeout_wrapper 30 codex exec "x" -s read-only',
|
||||
'```',
|
||||
].join('\n');
|
||||
const sites = codexSites(doc, 'planted.md');
|
||||
expect(sites.map(s => s.line)).toEqual(['_gstack_codex_timeout_wrapper 30 codex exec "x" -s read-only']);
|
||||
expect(VALIDATOR.test(sites[0]!.after)).toBe(false);
|
||||
expect(sites[0]!.line.includes('${_GSTACK_CODEX_SANDBOX:?}')).toBe(false);
|
||||
expect(codexSites('```bash\nX=$(codex exec "y")\n```', 'p.md')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -28,11 +28,22 @@ import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { codexPreflight } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
|
||||
const SANDBOX_FIXTURES = path.join(ROOT, 'test', 'fixtures', 'codex-sandbox');
|
||||
const STUB = `#!/usr/bin/env bash
|
||||
if [ "$1" = sandbox ]; then
|
||||
echo "$*" >> "$STUB_LOG.sandbox"
|
||||
case "\${STUB_SANDBOX:-ok}" in
|
||||
ok) exit 0 ;;
|
||||
userns) cat "$SANDBOX_FIXTURES/sandbox-userns-denied.stderr" >&2; exit 1 ;;
|
||||
missing) cat "$SANDBOX_FIXTURES/sandbox-bwrap-missing.stderr" >&2; exit 101 ;;
|
||||
unknown) echo "error: unrecognized subcommand 'sandbox'" >&2; exit 2 ;;
|
||||
esac
|
||||
fi
|
||||
echo "invoked" >> "$STUB_LOG"
|
||||
printf '%s\\n' "$*" >> "$STUB_ARGS_LOG"
|
||||
case "\${STUB_MODE:-ok}" in
|
||||
@@ -42,6 +53,13 @@ case "\${STUB_MODE:-ok}" in
|
||||
echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-6-astra'"'"' model is not supported when using Codex with a ChatGPT account."}}' >&2
|
||||
exit 1 ;;
|
||||
transient) echo "stream error: network unreachable" >&2; exit 7 ;;
|
||||
retired404)
|
||||
echo 'ERROR: unexpected status 404 Not Found: The model \`gpt-5.2-codex\` does not exist or you do not have access to it., url: https://chatgpt.com/backend-api/codex/responses' >&2
|
||||
exit 1 ;;
|
||||
baseurl404) echo 'ERROR: unexpected status 404 Not Found: <html><body>Not Found</body></html>, url: http://localhost:9999/v1/responses' >&2; exit 1 ;;
|
||||
outdated400)
|
||||
echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-6-astra'"'"' model requires a newer version of Codex. Please upgrade to the latest app or CLI and try again."}}' >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
`;
|
||||
|
||||
@@ -83,6 +101,7 @@ function runProbe(f: Fixture, stubMode: string, extraEnv: Record<string, string>
|
||||
STUB_MODE: stubMode,
|
||||
STUB_LOG: f.stubLog,
|
||||
STUB_ARGS_LOG: f.stubArgsLog,
|
||||
SANDBOX_FIXTURES,
|
||||
_TEL: 'off',
|
||||
...extraEnv,
|
||||
},
|
||||
@@ -339,3 +358,144 @@ describe('codex model probe (#2477)', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('B1: codex sandbox preflight and unverified readiness', () => {
|
||||
const unavailable = (detail: string) =>
|
||||
`Codex outside review unavailable: Codex's sandbox could not start here (${detail}). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.`;
|
||||
|
||||
for (const [mode, detail] of [
|
||||
['userns', 'bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.'.slice(0, 240)],
|
||||
['missing', 'bubblewrap is unavailable: no system bwrap was found on PATH and no bundled codex-resources/bwrap binary was found next to the Codex executable'],
|
||||
] as const) {
|
||||
test(`captured ${mode} failure -> sandbox unavailable (exit 3), named like the gate outcome`, () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
const r = runProbe(f, 'ok', { STUB_SANDBOX: mode }, '_gstack_codex_sandbox_preflight; echo "rc=$?"');
|
||||
expect(r.stdout).toBe('CODEX_SANDBOX: unavailable\nrc=3\n');
|
||||
expect(r.stderr.trim()).toBe(unavailable(detail));
|
||||
expect(fs.readFileSync(`${f.stubLog}.sandbox`, 'utf-8')).toBe(`sandbox -c sandbox_mode="read-only" true\n`);
|
||||
expect(invocations(f)).toBe(0);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
test('a healthy sandbox, an older CLI without the subcommand, or a timeout defers to the post-run check', () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
for (const mode of ['ok', 'unknown']) {
|
||||
const r = runProbe(f, 'ok', { STUB_SANDBOX: mode }, '_gstack_codex_sandbox_preflight; echo "rc=$?"');
|
||||
expect(r.stdout).toBe('rc=0\n');
|
||||
expect(r.stderr).toBe('');
|
||||
}
|
||||
const timedOut = runProbe(f, 'ok', { STUB_SANDBOX: 'userns' },
|
||||
'_gstack_codex_timeout_wrapper() { return 124; }; _gstack_codex_sandbox_preflight; echo "rc=$?"');
|
||||
expect(timedOut.stdout).toBe('rc=0\n');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('GSTACK_CODEX_NO_SANDBOX: exactly 1 selects full access with a warning each use; other values keep read-only', () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
const warning = 'WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust.';
|
||||
const on = runProbe(f, 'ok', { GSTACK_CODEX_NO_SANDBOX: '1', STUB_SANDBOX: 'userns' },
|
||||
'_gstack_codex_select_model exec; echo "sandbox=$_GSTACK_CODEX_SANDBOX"; _gstack_codex_select_model exec; _gstack_codex_sandbox_preflight; echo "rc=$?"');
|
||||
expect(on.stdout).toBe('sandbox=danger-full-access\nrc=0\n');
|
||||
expect(on.stderr.split(warning).length - 1).toBe(2);
|
||||
expect(fs.existsSync(`${f.stubLog}.sandbox`)).toBe(false);
|
||||
for (const value of ['', 'true', 'yes', '0', '1 ']) {
|
||||
const off = runProbe(f, 'ok', { GSTACK_CODEX_NO_SANDBOX: value }, '_gstack_codex_select_model exec; echo "sandbox=$_GSTACK_CODEX_SANDBOX"');
|
||||
expect(off.stdout).toBe('sandbox=read-only\n');
|
||||
expect(off.stderr).not.toContain('WARNING');
|
||||
}
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('the preflight runs only on Linux', () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
fs.writeFileSync(path.join(f.stubDir, 'uname'), '#!/usr/bin/env bash\necho Darwin\n', { mode: 0o755 });
|
||||
const r = runProbe(f, 'ok', { STUB_SANDBOX: 'userns' }, '_gstack_codex_sandbox_preflight; echo "rc=$?"');
|
||||
expect(r.stdout).toBe('rc=0\n');
|
||||
expect(fs.existsSync(`${f.stubLog}.sandbox`)).toBe(false);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('an inconclusive model probe marks readiness unverified without blocking stale callers', () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
const r = runProbe(f, 'transient', {}, '_gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"');
|
||||
expect(r.stdout).toContain('CODEX_MODE: unverified');
|
||||
expect(r.stdout).toContain('rc=0 state=inconclusive');
|
||||
const ok = runProbe(f, 'ok', {}, '_gstack_codex_model_probe; echo "state=${_GSTACK_CODEX_PROBE_STATE:-none}"');
|
||||
expect(ok.stdout).toContain('state=none');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('the rendered preflight reports sandbox_unavailable before any paid probe, and unverified on an inconclusive probe', () => {
|
||||
const block = codexPreflight({ disabledBehavior: 'codex-only' }).match(/```bash\n([\s\S]*?)\n```/)![1];
|
||||
for (const [sandbox, model, mode, probed] of [
|
||||
['userns', 'ok', 'sandbox_unavailable', 0],
|
||||
['ok', 'transient', 'unverified', 1],
|
||||
['ok', 'ok', 'ready', 1],
|
||||
['ok', 'model400', 'model_unusable', 1],
|
||||
] as const) {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
fs.mkdirSync(path.join(f.home, '.claude', 'skills'), { recursive: true });
|
||||
fs.symlinkSync(ROOT, path.join(f.home, '.claude', 'skills', 'gstack'));
|
||||
const r = spawnSync('bash', ['-c', block], { encoding: 'utf8', timeout: 20000, env: {
|
||||
PATH: `${f.stubDir}:${process.env.PATH ?? ''}`, HOME: f.home, CODEX_HOME: f.codexHome, GSTACK_HOME: f.gstackHome,
|
||||
STUB_MODE: model, STUB_SANDBOX: sandbox, STUB_LOG: f.stubLog, STUB_ARGS_LOG: f.stubArgsLog, SANDBOX_FIXTURES } });
|
||||
expect(r.stdout).toContain(`CODEX_MODE: ${mode}`);
|
||||
const paid = fs.existsSync(f.stubArgsLog) ? fs.readFileSync(f.stubArgsLog, 'utf-8').split('\n').filter(l => l.includes('reply OK')).length : 0;
|
||||
expect(paid).toBe(probed);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('B2: a 404 model rejection is unusable, never ready (#2843)', () => {
|
||||
for (const [mode, hint, event] of [
|
||||
['retired404', "Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url", 'codex_model_retired'],
|
||||
['baseurl404', "a custom provider's base_url", 'codex_model_retired'],
|
||||
['outdated400', 'this Codex CLI is too old', 'codex_cli_outdated'],
|
||||
] as const) {
|
||||
test(`${mode} -> MODEL_UNUSABLE with a hint naming the cause and a ${event} event`, () => {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
const r = runProbe(f, mode, { _TEL: 'community' }, '_gstack_codex_model_probe; echo "rc=$?"');
|
||||
expect(r.stdout).toContain('MODEL_UNUSABLE');
|
||||
expect(r.stdout).toContain('rc=1');
|
||||
expect(r.stdout).not.toContain('MODEL_PROBE_INCONCLUSIVE');
|
||||
expect(r.stdout).toContain(hint);
|
||||
expect(fs.readFileSync(path.join(f.gstackHome, 'analytics', 'skill-usage.jsonl'), 'utf-8')).toContain(`"event":"${event}"`);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('Q2: one-time outside-review notice (#965)', () => {
|
||||
test('names the account source per auth mode, never the credential, and shows once per state root', () => {
|
||||
for (const [env, files, account] of [
|
||||
[{ CODEX_API_KEY: 'sk-codex-secret' }, {}, 'the API key in CODEX_API_KEY'],
|
||||
[{ OPENAI_API_KEY: 'sk-openai-secret' }, {}, 'the API key in OPENAI_API_KEY'],
|
||||
[{}, { 'auth.json': '{"auth_mode":"chatgpt","tokens":{"id_token":"secret-token"}}' }, 'the ChatGPT account Codex is logged in with'],
|
||||
[{}, { 'auth.json': '{"auth_mode":"apikey","OPENAI_API_KEY":"sk-file-secret"}' }, 'the API key saved by codex login'],
|
||||
[{}, { 'config.toml': 'model_provider = "mimo"\n[model_providers.mimo]\nenv_key = "MIMO_API_KEY"\n' }, 'Codex (provider: mimo) using the credentials your Codex config.toml provider names'],
|
||||
] as const) {
|
||||
const f = makeFixture();
|
||||
try {
|
||||
fs.rmSync(path.join(f.codexHome, 'auth.json'));
|
||||
fs.rmSync(path.join(f.codexHome, 'config.toml'));
|
||||
for (const [name, body] of Object.entries(files)) fs.writeFileSync(path.join(f.codexHome, name), body);
|
||||
const first = runProbe(f, 'ok', { ...env }, '_gstack_codex_first_use_notice; echo "rc=$?"');
|
||||
expect(first.stdout).toBe('rc=0\n');
|
||||
expect(first.stderr).toContain(account);
|
||||
expect(first.stderr).toContain('To turn them off: gstack-config set codex_reviews disabled');
|
||||
expect(first.stderr).not.toMatch(/secret/);
|
||||
expect(fs.existsSync(path.join(f.gstackHome, '.codex-review-notice-shown'))).toBe(true);
|
||||
expect(runProbe(f, 'ok', { ...env }, '_gstack_codex_first_use_notice').stderr).toBe('');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
}
|
||||
});
|
||||
});
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
# Codex sandbox fixtures (B1)
|
||||
|
||||
Captured from codex-cli 0.160.0 (musl x86_64 vendor binary). They pin what
|
||||
`lib/outside-review-result.ts` and `_gstack_codex_sandbox_preflight` classify.
|
||||
|
||||
| File | Capture | Exit |
|
||||
|---|---|---|
|
||||
| `sandbox-userns-denied.stderr` | `codex sandbox -c 'sandbox_mode="read-only"' true` in Docker `ubuntu:24.04`, default seccomp profile, apt `bubblewrap` installed | 1 |
|
||||
| `sandbox-bwrap-missing.stderr` | same command without `bubblewrap` installed (Rust panic) | 101 |
|
||||
| `exec-json-userns-denied.jsonl` | `echo "Run cat a.txt and tell me its content in one word. If the command fails, say exactly what failed." \| codex exec - -s read-only -c 'model="gpt-5.4-mini"' -c skills.include_instructions=false --json` in the same container (stderr empty) | 0 |
|
||||
| `review-userns-denied.stdout` / `.stderr` | `codex review --base master -c 'model="gpt-5.4-mini"' -c 'review_model="gpt-5.4-mini"' -c 'sandbox_mode="read-only"' </dev/null` in the same container, on a branch with one changed line | 0 |
|
||||
| `review-healthy.stdout` / `.stderr` | the same review on a host with user namespaces enabled | 0 |
|
||||
| `exec-json-healthy.jsonl` | `echo 'Run \`cat a.txt\` and tell me its content in one word.' \| codex exec - -s read-only ... --json` on the same host | 0 |
|
||||
| `review-mentions-sandbox.txt` | constructed: a real review that discusses bwrap, namespaces, landlock and seccomp (must still pass) | - |
|
||||
|
||||
Both failing runs exit 0 and the review's own text concludes "there are no
|
||||
findings to report", which is why execution evidence (stderr transcript,
|
||||
`--json` command events) decides before any review text is trusted.
|
||||
|
||||
Reproduce the container captures:
|
||||
|
||||
```bash
|
||||
docker run --rm -v <codex vendor bin dir>:/cx:ro -v "$CODEX_HOME":/codexhome -e CODEX_HOME=/codexhome ubuntu:24.04 sh -c \
|
||||
'apt-get update -qq && apt-get install -y -qq bubblewrap git; PATH=/cx:$PATH codex sandbox -c "sandbox_mode=\"read-only\"" true; echo $?'
|
||||
```
|
||||
|
||||
Omit `bubblewrap` from the install for the exit-101 capture. Host paths in the
|
||||
healthy captures are normalized to `/repo`, and lines written by the capture
|
||||
machine's git wrapper are removed.
|
||||
@@ -0,0 +1,8 @@
|
||||
{"type":"thread.started","thread_id":"01a10376-d6d1-7b60-8c14-6decd95f8437"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"error","message":"Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."}}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"I’ll read `a.txt` and report it in one word."}}
|
||||
{"type":"item.started","item":{"id":"item_2","type":"command_execution","command":"/bin/bash -c 'cat a.txt'","aggregated_output":"","exit_code":null,"status":"in_progress"}}
|
||||
{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"/bin/bash -c 'cat a.txt'","aggregated_output":"hello\n","exit_code":0,"status":"completed"}}
|
||||
{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"hello"}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":22391,"cached_input_tokens":11136,"cache_write_input_tokens":0,"output_tokens":85,"reasoning_output_tokens":0}}
|
||||
@@ -0,0 +1,8 @@
|
||||
{"type":"thread.started","thread_id":"01a10377-a0ec-7b01-81f6-384bd91a865b"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"error","message":"Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."}}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"I’ll check `a.txt` and report the result exactly."}}
|
||||
{"type":"item.started","item":{"id":"item_2","type":"command_execution","command":"/bin/bash -lc 'cat a.txt'","aggregated_output":"","exit_code":null,"status":"in_progress"}}
|
||||
{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"/bin/bash -lc 'cat a.txt'","aggregated_output":"bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.\n","exit_code":1,"status":"failed"}}
|
||||
{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces."}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":22392,"cached_input_tokens":11136,"cache_write_input_tokens":0,"output_tokens":89,"reasoning_output_tokens":0}}
|
||||
@@ -0,0 +1,31 @@
|
||||
OpenAI Codex v0.160.0
|
||||
--------
|
||||
workdir: /repo
|
||||
model: gpt-5.4-mini
|
||||
provider: openai
|
||||
approval: never
|
||||
sandbox: read-only
|
||||
reasoning effort: none
|
||||
reasoning summaries: none
|
||||
session id: 01a1037c-220d-7e32-816f-3dfb0b645a38
|
||||
--------
|
||||
user
|
||||
changes against 'master'
|
||||
warning: Codex could not find bubblewrap on PATH. Install bubblewrap with your OS package manager. See the sandbox prerequisites: https://developers.openai.com/codex/concepts/sandboxing#prerequisites. Codex will use the bundled bubblewrap in the meantime.
|
||||
warning: Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues.
|
||||
warning: Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues.
|
||||
exec
|
||||
/bin/bash -lc 'git diff e66c79820df46b1a805e79ffe69e1c0ea74e018c --stat && git diff e66c79820df46b1a805e79ffe69e1c0ea74e018c --' in /repo
|
||||
succeeded in 0ms:
|
||||
a.txt | 1 +
|
||||
1 file changed, 1 insertion(+)
|
||||
diff --git a/a.txt b/a.txt
|
||||
index ce01362..94954ab 100644
|
||||
--- a/a.txt
|
||||
+++ b/a.txt
|
||||
@@ -1 +1,2 @@
|
||||
hello
|
||||
+world
|
||||
|
||||
codex
|
||||
The diff only adds a new line to a text file and does not introduce any code-path changes or functional regressions. No actionable defects are present in the patch.
|
||||
@@ -0,0 +1 @@
|
||||
The diff only adds a new line to a text file and does not introduce any code-path changes or functional regressions. No actionable defects are present in the patch.
|
||||
@@ -0,0 +1,2 @@
|
||||
[P2] lib/sandbox.ts:41 — the bwrap fallback swallows "bwrap: No permissions to create new namespace" when user namespaces are disabled, and the landlock error path is untested.
|
||||
Recommendation: surface the sandbox error because a silent seccomp or landlock failure hides missing coverage.
|
||||
@@ -0,0 +1,37 @@
|
||||
OpenAI Codex v0.160.0
|
||||
--------
|
||||
workdir: /r
|
||||
model: gpt-5.4-mini
|
||||
provider: openai
|
||||
approval: never
|
||||
sandbox: read-only
|
||||
reasoning effort: none
|
||||
reasoning summaries: none
|
||||
session id: 01a10377-a9a4-7802-9b30-b415edbc0bfb
|
||||
--------
|
||||
user
|
||||
changes against 'master'
|
||||
warning: Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues.
|
||||
warning: Model metadata for `gpt-5.4-mini` not found. Defaulting to fallback metadata; this can degrade performance and cause issues.
|
||||
exec
|
||||
/bin/bash -lc 'git diff 98eb804c33b7c646f099ba6a3ba89ca40a3e28b7 --stat && git diff 98eb804c33b7c646f099ba6a3ba89ca40a3e28b7 --' in /r
|
||||
exited 1 in 0ms:
|
||||
bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.
|
||||
|
||||
exec
|
||||
/bin/bash -c "/bin/bash -lc 'cd /r && git diff 98eb804c33b7c646f099ba6a3ba89ca40a3e28b7 --stat && git diff 98eb804c33b7c646f099ba6a3ba89ca40a3e28b7 --'" in /r
|
||||
exited 1 in 0ms:
|
||||
bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.
|
||||
|
||||
exec
|
||||
/bin/bash -c 'pwd && ls -la && git status --short && git rev-parse --show-toplevel' in /r
|
||||
exited 1 in 0ms:
|
||||
bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.
|
||||
|
||||
exec
|
||||
/bin/bash -c 'ls /r' in /
|
||||
exited 1 in 0ms:
|
||||
bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.
|
||||
|
||||
codex
|
||||
I was unable to inspect the diff because every shell invocation failed immediately with the environment's bubblewrap/user-namespace restriction, so no concrete regressions could be identified from the provided patch. With no verifiable issues in the changed code, there are no findings to report.
|
||||
@@ -0,0 +1 @@
|
||||
I was unable to inspect the diff because every shell invocation failed immediately with the environment's bubblewrap/user-namespace restriction, so no concrete regressions could be identified from the provided patch. With no verifiable issues in the changed code, there are no findings to report.
|
||||
@@ -0,0 +1,5 @@
|
||||
WARNING: proceeding, even though we could not create PATH aliases: Refusing to create helper binaries under temporary dir "/tmp" (codex_home: AbsolutePathBuf("/tmp/.codex"))
|
||||
|
||||
thread 'main' (15) panicked at linux-sandbox/src/launcher.rs:57:13:
|
||||
bubblewrap is unavailable: no system bwrap was found on PATH and no bundled codex-resources/bwrap binary was found next to the Codex executable
|
||||
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
||||
@@ -0,0 +1 @@
|
||||
bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces. See <https://deb.li/bubblewrap> or <file:///usr/share/doc/bubblewrap/README.Debian.gz>.
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* INV-1 lint: a check that could not run must never look like "nothing found".
|
||||
* 1. The learnings/timeline query scripts may not end in `2>/dev/null || exit 0`
|
||||
* (a missing bun read as "no learnings recorded").
|
||||
* 2. No template or resolver may wrap a gate binary in `|| true`, which turns
|
||||
* "could not run" into success.
|
||||
* 3. Any other `2>/dev/null || exit 0` tail in bin/ is a deliberate best-effort
|
||||
* exit and says why on the same line (`# best-effort: <reason>`).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const QUERY_SCRIPTS = ['bin/gstack-learnings-search', 'bin/gstack-timeline-read'];
|
||||
const GATE_CALLS = ['outside-review-result.ts', 'gstack-learnings-search', 'gstack-timeline-read',
|
||||
'_gstack_codex_auth_probe', '_gstack_codex_model_probe', '_gstack_codex_sandbox_preflight'];
|
||||
const SILENT_EXIT = /2>\/dev\/null \|\| exit 0\b/;
|
||||
|
||||
function walk(dir: string, keep: (file: string) => boolean, out: string[] = []): string[] {
|
||||
for (const entry of fs.readdirSync(path.join(ROOT, dir), { withFileTypes: true })) {
|
||||
if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
|
||||
const rel = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(rel, keep, out);
|
||||
else if (keep(rel)) out.push(rel);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function sources(): string[] {
|
||||
const templates = fs.readdirSync(ROOT, { withFileTypes: true })
|
||||
.filter(d => d.isDirectory() && !d.name.startsWith('.') && !['node_modules', 'test'].includes(d.name))
|
||||
.flatMap(d => walk(d.name, f => f.endsWith('.tmpl')));
|
||||
return [...templates, ...walk('scripts/resolvers', f => f.endsWith('.ts')), ...walk('bin', () => true)];
|
||||
}
|
||||
|
||||
describe('gate tails never hide a check that did not run', () => {
|
||||
test('query scripts do not swallow a failed run as an empty result', () => {
|
||||
const offenders = QUERY_SCRIPTS.filter(f => SILENT_EXIT.test(fs.readFileSync(path.join(ROOT, f), 'utf8')));
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
test('no gate binary call is wrapped in || true', () => {
|
||||
const offenders: string[] = [];
|
||||
for (const rel of sources()) {
|
||||
fs.readFileSync(path.join(ROOT, rel), 'utf8').split('\n').forEach((line, i) => {
|
||||
if (/^\s*(?:#|\/\/|\*)/.test(line)) return;
|
||||
if (GATE_CALLS.some(g => line.includes(g)) && /\|\|\s*true\b/.test(line)) offenders.push(`${rel}:${i + 1}: ${line.trim()}`);
|
||||
});
|
||||
}
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
test('other best-effort exits in bin/ carry a same-line reason', () => {
|
||||
const offenders: string[] = [];
|
||||
for (const rel of walk('bin', () => true)) {
|
||||
if (QUERY_SCRIPTS.includes(rel)) continue;
|
||||
fs.readFileSync(path.join(ROOT, rel), 'utf8').split('\n').forEach((line, i) => {
|
||||
if (SILENT_EXIT.test(line) && !/#\s*best-effort:\s*\S/.test(line)) offenders.push(`${rel}:${i + 1}: ${line.trim()}`);
|
||||
});
|
||||
}
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
test('negative controls: each rule flags a planted offender', () => {
|
||||
expect(SILENT_EXIT.test('" 2>/dev/null || exit 0')).toBe(true);
|
||||
expect(/#\s*best-effort:\s*\S/.test('mkdir -p "$X" 2>/dev/null || exit 0 # best-effort: spool is optional')).toBe(true);
|
||||
const line = '$GSTACK_BIN/gstack-learnings-search --limit 10 2>/dev/null || true';
|
||||
expect(GATE_CALLS.some(g => line.includes(g)) && /\|\|\s*true\b/.test(line)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -2193,16 +2193,19 @@ describe('DESIGN_OUTSIDE_VOICES resolver', () => {
|
||||
const marker = path.join(dir, 'must-not-execute');
|
||||
const product = `A product with $(touch ${marker}), \`touch ${marker}\`, and "quotes".\nUsers: builders.\n`;
|
||||
fs.writeFileSync(brief, product);
|
||||
fs.writeFileSync(path.join(dir, 'codex'), `#!${process.execPath}\nrequire('fs').writeFileSync(process.env.CAPTURE, JSON.stringify(process.argv.slice(2)));\nconsole.log('Recommendation: choose a clear hierarchy because builders need to find their work.');\n`, { mode: 0o700 });
|
||||
// Fake codex: the free sandbox preflight succeeds; exec writes its final message to -o.
|
||||
fs.writeFileSync(path.join(dir, 'codex'), `#!${process.execPath}\nconst fs = require('fs'); const a = process.argv.slice(2); if (a[0] === 'sandbox') process.exit(0);\nfs.writeFileSync(process.env.CAPTURE, JSON.stringify({ args: a, stdin: a[1] === '-' ? fs.readFileSync(0, 'utf8') : '' }));\nconst msg = 'Recommendation: choose a clear hierarchy because builders need to find their work.';\nif (a.includes('-o')) fs.writeFileSync(a[a.indexOf('-o') + 1], msg);\nconsole.log(msg);\n`, { mode: 0o700 });
|
||||
const prepare = (file: string) => command.replace("'<prepared-prompt-file>'", quote(file))
|
||||
.replaceAll('$HOME/.claude/skills/gstack', ROOT);
|
||||
const env = { ...process.env, PATH: dir + path.delimiter + process.env.PATH, CAPTURE: capture,
|
||||
CODEX_THREAD_ID: '', CODEX_SANDBOX: '', CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' };
|
||||
CODEX_THREAD_ID: '', CODEX_SANDBOX: '', CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude',
|
||||
GSTACK_HOME: path.join(dir, 'state'), GSTACK_STATE_ROOT: '' };
|
||||
const result = spawnSync('bash', ['-c', prepare(brief)], { cwd: ROOT, env, encoding: 'utf8', timeout: 5_000 });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const args = JSON.parse(fs.readFileSync(capture, 'utf8'));
|
||||
const { args, stdin } = JSON.parse(fs.readFileSync(capture, 'utf8'));
|
||||
expect(args[0]).toBe('exec');
|
||||
expect(args[1]).toBe(product.trimEnd());
|
||||
expect(args[1]).toBe('-');
|
||||
expect(stdin).toBe(product);
|
||||
expect(args).toContain('read-only');
|
||||
expect(fs.readFileSync(brief, 'utf8')).toBe(product);
|
||||
expect(fs.existsSync(marker)).toBe(false);
|
||||
|
||||
@@ -247,9 +247,10 @@ export function autoplanDualVoiceEvidence(transcript: unknown[], options: Autopl
|
||||
if (execution.succeeded && /^OUTSIDE_STATUS: completed provider=codex host=claude$/m.test(execution.output)) {
|
||||
result.codexAttempted = result.codexVoiceFired = true; result.outsideToolUseId = outside.id;
|
||||
} else if (!execution.background && !execution.succeeded && outside.result.error &&
|
||||
/^(?:Codex outside review unavailable: execution failed; missing coverage\. Check the provider diagnosis above\.|Outside review unavailable: (?:empty response|review refused|missing review completion recommendation); missing coverage\.)$/m.test(execution.output)) {
|
||||
/^(?:Codex outside review unavailable: execution failed; missing coverage\. Check the provider diagnosis above\.|Outside review unavailable: (?:empty response|review refused|missing review completion recommendation); missing coverage\.|Codex outside review unavailable: [^\n]+\. No review ran; this is missing coverage, not a pass\. Fix: [^\n]+)$/m.test(execution.output)) {
|
||||
// These finite diagnostics occur only after the exact generated
|
||||
// command reaches Codex or its review validator. They prove an
|
||||
// command reaches Codex or its review validator (lib/gate-outcomes.ts
|
||||
// lines from the verdict form; the first two are pre-verdict renders). They prove an
|
||||
// attempted voice, never a completed review or comparison.
|
||||
result.codexAttempted = result.codexUnavailable = true;
|
||||
result.failedOutsideToolUseId = outside.id;
|
||||
@@ -258,7 +259,7 @@ export function autoplanDualVoiceEvidence(transcript: unknown[], options: Autopl
|
||||
}
|
||||
}
|
||||
}
|
||||
result.codexUnavailable ||= result.claudeVoiceFired && ['not_installed', 'not_authed', 'broken_install', 'model_unusable'].includes(result.probeMode ?? '');
|
||||
result.codexUnavailable ||= result.claudeVoiceFired && ['not_installed', 'not_authed', 'broken_install', 'sandbox_unavailable', 'model_unusable'].includes(result.probeMode ?? '');
|
||||
if (!result.claudeVoiceFired) result.reasons.push('No acknowledged current CEO phase dispatch');
|
||||
if (!result.codexVoiceFired && !result.codexUnavailable) {
|
||||
result.reasons.push('No acknowledged outside execution or actual unavailable probe result');
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
/**
|
||||
* INV-1 outside-review contract: classifyOutsideReview() separates execution,
|
||||
* findings and verdict; validateOutsideReview() and the two-argument CLI keep
|
||||
* their old shape for importers and installed skills that predate a re-render.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { classifyOutsideReview, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const LIB = path.join(ROOT, 'lib', 'outside-review-result.ts');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'outside-review-result-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
let n = 0;
|
||||
function file(content: string): string {
|
||||
const p = path.join(TMP, `f${n++}.txt`);
|
||||
fs.writeFileSync(p, content);
|
||||
return p;
|
||||
}
|
||||
function cli(args: string[]) {
|
||||
const r = spawnSync(process.execPath, [LIB, ...args], { encoding: 'utf8', timeout: 10000 });
|
||||
return { status: r.status, stdout: r.stdout, stderr: r.stderr };
|
||||
}
|
||||
|
||||
const RECOMMEND = 'Recommendation: fix the guard because changed.ts loses data.';
|
||||
|
||||
describe('classifyOutsideReview: separate execution, findings and verdict', () => {
|
||||
const cases: Array<[string, { text: string; gate: OutsideGate; stderr?: string; exit?: number }, string, string | null, string | undefined]> = [
|
||||
['clean review', { text: RECOMMEND, gate: 'review' }, 'clean', null, undefined],
|
||||
['review with P1', { text: `[P1] data loss\n${RECOMMEND}`, gate: 'review' }, 'findings', 'P1', undefined],
|
||||
['review with only P2', { text: `[P2] naming\n${RECOMMEND}`, gate: 'review' }, 'clean', 'P2', undefined],
|
||||
['review missing recommendation', { text: 'A few observations.', gate: 'review' }, 'unavailable', null, 'missing_markers'],
|
||||
['structured untagged prose', { text: 'The change looks reasonable overall.', gate: 'structured' }, 'unverified', null, 'untagged_review'],
|
||||
['structured explicit clear', { text: 'NO_FINDINGS', gate: 'structured' }, 'clean', null, undefined],
|
||||
['structured native P1 label', { text: 'P1: race in cleanup', gate: 'structured' }, 'findings', 'P1', undefined],
|
||||
['spec passing score', { text: 'SCORE: 8\nAMBIGUITIES: NONE', gate: 'spec' }, 'clean', null, undefined],
|
||||
['spec failing score', { text: 'SCORE: 4\nAMBIGUITIES: scope', gate: 'spec' }, 'findings', null, undefined],
|
||||
['nonzero exit', { text: RECOMMEND, gate: 'review', exit: 1, stderr: '\nerror: 401 Unauthorized\n' }, 'unavailable', null, 'execution_failed'],
|
||||
['timeout', { text: 'Partial', gate: 'review', exit: 124 }, 'unavailable', null, 'timeout'],
|
||||
['empty', { text: ' \n', gate: 'structured' }, 'unavailable', null, 'empty_response'],
|
||||
['refusal', { text: `I cannot review this request. ${RECOMMEND}`, gate: 'review' }, 'unavailable', null, 'review_refused'],
|
||||
['consult answer without review markers', { text: 'Use a queue here; the writer pool blocks.', gate: 'execution', stderr: '' }, 'clean', null, undefined],
|
||||
];
|
||||
for (const [label, input, verdict, highest, reason] of cases) {
|
||||
test(label, () => {
|
||||
const result = classifyOutsideReview(input);
|
||||
expect(result.verdict).toBe(verdict as any);
|
||||
expect(result.findings.highest).toBe(highest as any);
|
||||
expect(result.reason).toBe(reason as any);
|
||||
expect(result.execution.state).toBe(['execution_failed', 'timeout', 'empty_response', 'review_refused'].includes(reason ?? '') ? 'unavailable' : 'ran');
|
||||
});
|
||||
}
|
||||
|
||||
test('a non-zero exit names the exit code and the first stderr line', () => {
|
||||
const result = classifyOutsideReview({ text: '', gate: 'review', exit: 1, stderr: '\nerror: 401 Unauthorized\nmore' });
|
||||
expect(result.detail).toBe('exit 1: error: 401 Unauthorized');
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateOutsideReview keeps the text-only shape for direct importers', () => {
|
||||
test('existing shapes are unchanged', () => {
|
||||
expect(validateOutsideReview('', 'review')).toEqual({ completed: false, reason: 'empty response' });
|
||||
expect(validateOutsideReview('I must refuse.', 'review')).toEqual({ completed: false, reason: 'review refused' });
|
||||
expect(validateOutsideReview('notes', 'review')).toEqual({ completed: false, reason: 'missing review completion recommendation' });
|
||||
expect(validateOutsideReview('notes', 'structured')).toEqual({ completed: false, reason: 'missing severity or explicit no-findings conclusion' });
|
||||
expect(validateOutsideReview('SCORE: 8', 'spec')).toEqual({ completed: false, reason: 'missing or invalid SCORE/AMBIGUITIES markers' });
|
||||
expect(validateOutsideReview('SCORE: 6\nAMBIGUITIES: NONE', 'spec')).toEqual({ completed: true, score: 6, gate: 'fail' });
|
||||
expect(validateOutsideReview(RECOMMEND, 'review')).toEqual({ completed: true });
|
||||
expect(validateOutsideReview('[P2] nit', 'structured')).toEqual({ completed: true, gate: 'pass' });
|
||||
expect(validateOutsideReview('[P1] bug', 'structured')).toEqual({ completed: true, gate: 'fail' });
|
||||
});
|
||||
|
||||
test('B1b: the structured gate fails on P0 as well as P1', () => {
|
||||
for (const text of ['[P0] corrupts every row', 'P0: corrupts every row', '**P0:** corrupts every row', '[P3] nit\n[P0] corrupts every row']) {
|
||||
expect(validateOutsideReview(text, 'structured')).toEqual({ completed: true, gate: 'fail' });
|
||||
}
|
||||
expect(validateOutsideReview('[P2] naming\n[P3] nit', 'structured')).toEqual({ completed: true, gate: 'pass' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('outside-review-result CLI', () => {
|
||||
test('two-argument form keeps exit 0 completed / 1 unavailable / 2 usage, findings included', () => {
|
||||
expect(cli(['structured', file('[P1] data loss')]).status).toBe(0);
|
||||
expect(cli(['review', file(`[P0] data loss\n${RECOMMEND}`)]).status).toBe(0);
|
||||
const untagged = cli(['structured', file('looks fine')]);
|
||||
expect(untagged.status).toBe(1);
|
||||
expect(untagged.stderr).toBe('Outside review unavailable: missing severity or explicit no-findings conclusion; missing coverage.\n');
|
||||
expect(cli(['structured', path.join(TMP, 'missing-file')]).status).toBe(1);
|
||||
expect(cli(['bogus', file('x')]).status).toBe(2);
|
||||
expect(cli(['execution', file('x')]).status).toBe(2);
|
||||
expect(cli(['review']).status).toBe(2);
|
||||
// No VERDICT line: stale callers only read the status.
|
||||
expect(cli(['structured', file('[P1] data loss')]).stdout).toBe('');
|
||||
});
|
||||
|
||||
test('verdict form prints VERDICT and exits 0 clean / 3 findings / 4 unverified / 1 unavailable', () => {
|
||||
const clean = cli(['--verdict', 'review', file(RECOMMEND)]);
|
||||
expect([clean.status, clean.stdout]).toEqual([0, 'VERDICT: clean\nFINDINGS: none\n']);
|
||||
const findings = cli(['--verdict', 'structured', file('[P0] corrupts data')]);
|
||||
expect([findings.status, findings.stdout]).toEqual([3, 'VERDICT: findings\nFINDINGS: P0\n']);
|
||||
const unverified = cli(['--verdict', 'structured', file('looks fine')]);
|
||||
expect(unverified.status).toBe(4);
|
||||
expect(unverified.stdout).toContain('VERDICT: unverified\n');
|
||||
expect(unverified.stderr).toContain('Outside review: ran, verdict unverified');
|
||||
const unavailable = cli(['--verdict', '--label', 'Codex outside review', 'review', file('')]);
|
||||
expect(unavailable.status).toBe(1);
|
||||
expect(unavailable.stdout).toContain('VERDICT: unavailable\nFINDINGS: none\nREASON: empty_response\n');
|
||||
expect(unavailable.stderr).toMatch(/^Codex outside review unavailable: the reviewer returned no response\. .*Fix: /);
|
||||
});
|
||||
|
||||
test('--stderr and --exit imply the verdict form; a missing stderr file reads as empty', () => {
|
||||
const failed = cli(['--exit', '7', '--stderr', file('fatal: provider exploded\n'), 'review', file(RECOMMEND)]);
|
||||
expect(failed.status).toBe(1);
|
||||
expect(failed.stdout).toContain('REASON: execution_failed');
|
||||
expect(failed.stderr).toContain('(exit 7: fatal: provider exploded)');
|
||||
const missing = cli(['--stderr', path.join(TMP, 'never-written'), 'review', file(RECOMMEND)]);
|
||||
expect([missing.status, missing.stdout]).toEqual([0, 'VERDICT: clean\nFINDINGS: none\n']);
|
||||
expect(cli(['--exit', 'x', 'review', file(RECOMMEND)]).status).toBe(2);
|
||||
});
|
||||
|
||||
test('an old rendered caller (pre-verdict skill text) still records findings as completed', () => {
|
||||
// The tail every generated outside-review fence carried before the verdict
|
||||
// form; bin/ and lib/ run live from the checkout, rendered text may lag.
|
||||
const response = file('[P1] Seeded data-loss bug\nREVIEW_COMPLETE');
|
||||
const old = `bun "$LIB" structured "$RESPONSE" || exit 1\necho 'OUTSIDE_STATUS: completed provider=codex host=claude'`;
|
||||
const r = spawnSync('bash', ['-c', old], { encoding: 'utf8', timeout: 10000,
|
||||
env: { ...process.env, LIB, RESPONSE: response } });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toBe('OUTSIDE_STATUS: completed provider=codex host=claude\n');
|
||||
});
|
||||
});
|
||||
|
||||
describe('B1: a review whose sandbox could not start is unavailable, not clean', () => {
|
||||
const FIX = path.join(ROOT, 'test', 'fixtures', 'codex-sandbox');
|
||||
const read = (name: string) => fs.readFileSync(path.join(FIX, name), 'utf8');
|
||||
const BWRAP = 'bwrap: No permissions to create new namespace, likely because the kernel does not allow non-privileged user namespaces.';
|
||||
|
||||
test('captured codex review in a container: exit 0, "no findings" text, bwrap failures on stderr', () => {
|
||||
const result = classifyOutsideReview({ text: read('review-userns-denied.stdout'), stderr: read('review-userns-denied.stderr'), exit: 0, gate: 'structured' });
|
||||
expect(result.verdict).toBe('unavailable');
|
||||
expect(result.reason).toBe('sandbox_unavailable');
|
||||
expect(result.detail).toStartWith(BWRAP);
|
||||
// The text-only wrapper (old importers) also stops passing it: the review says it could not read the diff.
|
||||
expect(validateOutsideReview(read('review-userns-denied.stdout'), 'structured').completed).toBe(false);
|
||||
});
|
||||
|
||||
test('captured codex exec --json in a container: every command failed with a sandbox error', () => {
|
||||
const result = classifyOutsideReview({ text: 'hello', events: read('exec-json-userns-denied.jsonl'), gate: 'execution' });
|
||||
expect([result.verdict, result.reason]).toEqual(['unavailable', 'sandbox_unavailable']);
|
||||
expect(result.detail).toStartWith(BWRAP);
|
||||
});
|
||||
|
||||
test('captured healthy runs count as executed even though Codex warns about bubblewrap', () => {
|
||||
const review = classifyOutsideReview({ text: read('review-healthy.stdout'), stderr: read('review-healthy.stderr'), gate: 'structured' });
|
||||
expect(review.execution.state).toBe('ran');
|
||||
expect(read('review-healthy.stderr')).toContain('could not find bubblewrap on PATH');
|
||||
expect(classifyOutsideReview({ text: 'hello', events: read('exec-json-healthy.jsonl'), gate: 'execution' }).verdict).toBe('clean');
|
||||
});
|
||||
|
||||
test('a real review that discusses bwrap, namespaces, landlock and seccomp still passes', () => {
|
||||
const result = classifyOutsideReview({ text: read('review-mentions-sandbox.txt'), stderr: '', exit: 0, gate: 'review' });
|
||||
expect([result.verdict, result.findings.highest, result.execution.state]).toEqual(['clean', 'P2', 'ran']);
|
||||
});
|
||||
|
||||
test('sandbox stderr with a non-zero exit (preflight fixtures) names the sandbox line', () => {
|
||||
const denied = classifyOutsideReview({ text: '', stderr: read('sandbox-userns-denied.stderr'), exit: 1, gate: 'review' });
|
||||
expect([denied.reason, denied.detail?.slice(0, BWRAP.length)]).toEqual(['sandbox_unavailable', BWRAP]);
|
||||
const missing = classifyOutsideReview({ text: '', stderr: read('sandbox-bwrap-missing.stderr'), exit: 101, gate: 'review' });
|
||||
expect(missing.reason).toBe('sandbox_unavailable');
|
||||
expect(missing.detail).toStartWith('bubblewrap is unavailable');
|
||||
});
|
||||
|
||||
test('exact execution-failure phrases are the last fallback; positive evidence wins', () => {
|
||||
const probe = 'I could not run commands in this sandbox, so I reviewed nothing. No issues found.';
|
||||
expect(classifyOutsideReview({ text: probe, stderr: '', gate: 'structured' }).reason).toBe('commands_failed');
|
||||
expect(classifyOutsideReview({ text: `The diff could not be read.\n${RECOMMEND}`, gate: 'review' }).reason).toBe('commands_failed');
|
||||
const executed = '{"type":"item.completed","item":{"type":"command_execution","exit_code":0,"status":"completed","aggregated_output":"ok"}}';
|
||||
expect(classifyOutsideReview({ text: `If commands could not run, the helper retries.\n${RECOMMEND}`, events: executed, gate: 'review' }).verdict).toBe('clean');
|
||||
const failed = '{"type":"item.completed","item":{"type":"command_execution","exit_code":2,"status":"failed","aggregated_output":"fatal: not a git repository"}}';
|
||||
expect(classifyOutsideReview({ text: RECOMMEND, events: `${failed}\n${failed}`, gate: 'review' }).detail).toBe('all 2 commands failed');
|
||||
});
|
||||
|
||||
test('a clean "I did not find any issues" with empty stderr still passes', () => {
|
||||
expect(classifyOutsideReview({ text: 'I did not find any issues.', stderr: '', exit: 0, gate: 'structured' }).verdict).toBe('clean');
|
||||
});
|
||||
|
||||
test('the CLI reads --events', () => {
|
||||
const events = path.join(FIX, 'exec-json-userns-denied.jsonl');
|
||||
const r = cli(['--label', 'Codex outside review', '--events', events, 'execution', file('hello')]);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stdout).toContain('REASON: sandbox_unavailable');
|
||||
expect(r.stderr).toStartWith(`Codex outside review unavailable: Codex's sandbox could not start here (${BWRAP}`);
|
||||
});
|
||||
});
|
||||
@@ -24,8 +24,10 @@ fs.writeFileSync(PROMPT, PROMPT_TEXT);
|
||||
const fakeSource = `
|
||||
import {writeFileSync} from 'node:fs';
|
||||
const args = process.argv.slice(2);
|
||||
// The free sandbox preflight (\`codex sandbox ... true\`) succeeds unless a test plants its failure.
|
||||
if (args[0] === 'sandbox') { if (process.env.FAKE_SANDBOX_STDERR) { console.error(process.env.FAKE_SANDBOX_STDERR); process.exit(1); } process.exit(0); }
|
||||
const claude = process.env.FAKE_PROVIDER === 'claude-code';
|
||||
const prompt = claude ? await Bun.stdin.text() : args[0] === 'exec' ? args[1] : '';
|
||||
const prompt = claude || (args[0] === 'exec' && args[1] === '-') ? await Bun.stdin.text() : args[0] === 'exec' ? args[1] : '';
|
||||
writeFileSync(process.env.CAPTURE!, JSON.stringify({args,prompt,cwd:process.cwd()}));
|
||||
if (process.env.FAKE_MODE === 'timeout') {
|
||||
if (!claude) console.log('Partial finding before timeout');
|
||||
@@ -36,7 +38,13 @@ const response = process.env.FAKE_RESPONSE || 'Recommendation: fix the seeded de
|
||||
if (claude) {
|
||||
if (process.env.FAKE_MODE === 'malformed') {console.log('{broken');process.exit(0);}
|
||||
console.log(JSON.stringify({result:response,session_id:'outside-session',modelUsage:{'model-a':{inputTokens:4},'model-b':{inputTokens:8}}}));
|
||||
} else if (args.includes('-o')) {
|
||||
// codex exec --json -o <file>: events on stdout, the final message in <file>.
|
||||
writeFileSync(args[args.indexOf('-o') + 1], response);
|
||||
if (process.env.FAKE_EVENTS_FILE) process.stdout.write(await Bun.file(process.env.FAKE_EVENTS_FILE).text());
|
||||
else console.log(JSON.stringify({type:'item.completed',item:{type:'agent_message',text:response}}));
|
||||
} else console.log(response);
|
||||
if (process.env.FAKE_STDERR_FILE) process.stderr.write(await Bun.file(process.env.FAKE_STDERR_FILE).text());
|
||||
process.exit(process.env.FAKE_MODE === 'nonzero' ? 4 : 0);
|
||||
`;
|
||||
fs.writeFileSync(FAKE_CLAUDE, fakeSource);
|
||||
@@ -48,6 +56,7 @@ function environment(host: 'codex' | 'claude'): NodeJS.ProcessEnv {
|
||||
CAPTURE, FAKE_PROVIDER:host === 'codex' ? 'claude-code' : 'codex',
|
||||
CODEX_THREAD_ID:host === 'codex' ? 'codex-fixture' : '', CODEX_SANDBOX:'',
|
||||
CLAUDECODE:host === 'claude' ? '1' : '', GSTACK_ACTIVE_HOST:host, CODEX_HOME:TMP, GSTACK_CODEX_MODEL:'',
|
||||
GSTACK_HOME:path.join(TMP,'state'), GSTACK_STATE_ROOT:'', CODEX_API_KEY:'', OPENAI_API_KEY:'',
|
||||
PATH:`${BIN}${path.delimiter}${process.env.PATH}`,
|
||||
GIT_AUTHOR_NAME:'Test',GIT_AUTHOR_EMAIL:'test@example.invalid',GIT_COMMITTER_NAME:'Test',GIT_COMMITTER_EMAIL:'test@example.invalid'};
|
||||
}
|
||||
@@ -96,10 +105,9 @@ describe('generated outside-review dispatch', () => {
|
||||
test(`${host} dispatches the other CLI and keeps hostile prompt/path text literal`, () => {
|
||||
const result = invoke(host);
|
||||
expect(result.status).toBe(0);
|
||||
// Codex keeps its existing command-substitution argv contract, which
|
||||
// removes trailing LF; shell metacharacters within that value stay data.
|
||||
const expectedPrompt = host === 'codex' ? PROMPT_TEXT : PROMPT_TEXT.replace(/\n+$/,'');
|
||||
expect(capture().prompt).toBe(expectedPrompt);
|
||||
// Both CLIs read the prepared prompt on stdin, byte for byte (E5: no argv
|
||||
// size limit, no quoting); shell metacharacters stay data.
|
||||
expect(capture().prompt).toBe(PROMPT_TEXT);
|
||||
expect(capture().cwd).toBe(DIR);
|
||||
expect(fs.existsSync(path.join(DIR,'NEVER'))).toBe(false);
|
||||
expect(result.stdout).toContain(`OUTSIDE_STATUS: completed provider=${host === 'codex' ? 'claude-code' : 'codex'} host=${host}`);
|
||||
@@ -108,7 +116,7 @@ describe('generated outside-review dispatch', () => {
|
||||
expect(capture().args).not.toContain('--resume');
|
||||
expect(result.stdout).toContain('model-a');
|
||||
expect(result.stdout).toContain('model-b');
|
||||
} else expect(capture().args.slice(0,2)).toEqual(['exec',expectedPrompt]);
|
||||
} else expect(capture().args.slice(0,2)).toEqual(['exec','-']);
|
||||
});
|
||||
|
||||
test(`${host} rejects its own reviewer markers before any process starts`, () => {
|
||||
@@ -196,6 +204,94 @@ describe('generated outside-review dispatch', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('INV-1: the generated caller branches on VERDICT: findings complete, untagged is unverified', () => {
|
||||
for (const host of ['claude', 'codex'] as const) {
|
||||
const findings = invoke(host, { structuredBase: 'main', gate: 'structured' }, { FAKE_RESPONSE: '[P0] Seeded corruption' });
|
||||
expect(findings.status).toBe(0);
|
||||
expect(findings.stdout).toContain('VERDICT: findings\nFINDINGS: P0\n');
|
||||
expect(findings.stdout).toContain('OUTSIDE_STATUS: completed');
|
||||
const untagged = invoke(host, { structuredBase: 'main', gate: 'structured' }, { FAKE_RESPONSE: 'The change reads fine to me.' });
|
||||
expect(untagged.status).toBe(4);
|
||||
expect(untagged.stdout).toContain('VERDICT: unverified');
|
||||
expect(untagged.stdout).toContain('OUTSIDE_STATUS: unverified');
|
||||
expect(untagged.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
expect(untagged.stderr).toContain('ran, verdict unverified');
|
||||
}
|
||||
});
|
||||
|
||||
describe('B1: Codex sandbox failures are missing coverage, never a pass', () => {
|
||||
const FIX = path.join(ROOT, 'test', 'fixtures', 'codex-sandbox');
|
||||
const SANDBOX_LINE = "Codex outside review unavailable: Codex's sandbox could not start here (bwrap: No permissions to create new namespace";
|
||||
|
||||
test('the free preflight stops before any paid call', () => {
|
||||
const result = invoke('claude', {}, { FAKE_SANDBOX_STDERR: fs.readFileSync(path.join(FIX, 'sandbox-userns-denied.stderr'), 'utf8') });
|
||||
expect(result.status).toBe(1);
|
||||
expect(fs.existsSync(CAPTURE)).toBe(false);
|
||||
expect(result.stderr).toContain(SANDBOX_LINE);
|
||||
expect(result.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('exec: captured events where every command hit the sandbox are unavailable despite exit 0', () => {
|
||||
const result = invoke('claude', {}, { FAKE_EVENTS_FILE: path.join(FIX, 'exec-json-userns-denied.jsonl'),
|
||||
FAKE_RESPONSE: 'I could not run commands here. No issues found.\nRecommendation: ship because no issues were found.' });
|
||||
expect(result.status).toBe(1);
|
||||
expect(capture().args).toContain('--json');
|
||||
expect(result.stdout).toContain('REASON: sandbox_unavailable');
|
||||
expect(result.stderr).toContain(SANDBOX_LINE);
|
||||
expect(result.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('structured review: captured container transcript on stderr is unavailable, healthy transcript completes', () => {
|
||||
const failed = invoke('claude', { structuredBase: 'main', gate: 'structured' }, {
|
||||
FAKE_RESPONSE: fs.readFileSync(path.join(FIX, 'review-userns-denied.stdout'), 'utf8'),
|
||||
FAKE_STDERR_FILE: path.join(FIX, 'review-userns-denied.stderr') });
|
||||
expect(failed.status).toBe(1);
|
||||
expect(failed.stderr).toContain(SANDBOX_LINE);
|
||||
expect(failed.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
expect(capture().args).toContain('sandbox_mode="read-only"');
|
||||
const healthy = invoke('claude', { structuredBase: 'main', gate: 'structured' }, {
|
||||
FAKE_RESPONSE: '[P2] naming nit', FAKE_STDERR_FILE: path.join(FIX, 'review-healthy.stderr') });
|
||||
expect(healthy.status).toBe(0);
|
||||
expect(healthy.stdout).toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('GSTACK_CODEX_NO_SANDBOX=1 switches every site to full access, warns, and skips the preflight', () => {
|
||||
const env = { GSTACK_CODEX_NO_SANDBOX: '1', FAKE_SANDBOX_STDERR: 'bwrap: No permissions to create new namespace' };
|
||||
const exec = invoke('claude', {}, env);
|
||||
expect(exec.status).toBe(0);
|
||||
expect(capture().args.slice(capture().args.indexOf('-s'), capture().args.indexOf('-s') + 2)).toEqual(['-s', 'danger-full-access']);
|
||||
expect(exec.stderr).toContain('WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox');
|
||||
expect(invoke('claude', { structuredBase: 'main', gate: 'structured' }, { ...env, FAKE_RESPONSE: '[P2] nit' }).status).toBe(0);
|
||||
expect(capture().args).toContain('sandbox_mode="danger-full-access"');
|
||||
expect(invoke('claude', {}, { GSTACK_CODEX_NO_SANDBOX: 'true' }).status).toBe(0);
|
||||
expect(capture().args.slice(capture().args.indexOf('-s'), capture().args.indexOf('-s') + 2)).toEqual(['-s', 'read-only']);
|
||||
});
|
||||
});
|
||||
|
||||
test('E5: a prompt larger than one argv string reaches Codex intact on stdin', () => {
|
||||
const big = path.join(TMP, 'big-prompt.txt');
|
||||
const text = `${'x'.repeat(200_000)}\n"quotes" 'single' $(touch NEVER) \\ end\n`;
|
||||
fs.writeFileSync(big, text);
|
||||
const ctx: TemplateContext = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.codex };
|
||||
const result = spawnSync('bash', ['-c', outsideVoiceCommand(ctx, { promptFile: big, timeoutMs: 5000 })], { cwd: DIR, env: environment('claude'), encoding: 'utf8', timeout: 15000 });
|
||||
expect(result.status).toBe(0);
|
||||
expect(capture().prompt).toBe(text);
|
||||
expect(fs.existsSync(path.join(DIR, 'NEVER'))).toBe(false);
|
||||
});
|
||||
|
||||
test('Q2: the first automatic Codex review on a machine names the provider and account, once, without blocking', () => {
|
||||
const state = path.join(TMP, 'notice-state');
|
||||
const first = invoke('claude', {}, { GSTACK_HOME: state, OPENAI_API_KEY: 'sk-test-secret-value' });
|
||||
expect(first.status).toBe(0);
|
||||
expect(first.stderr).toContain('NOTICE: gstack outside reviews send the review prompt and code to Codex (provider: openai) using the API key in OPENAI_API_KEY.');
|
||||
expect(first.stderr).toContain('To turn them off: gstack-config set codex_reviews disabled');
|
||||
expect(first.stderr).not.toContain('sk-test-secret-value');
|
||||
expect(first.stdout).toContain('OUTSIDE_STATUS: completed');
|
||||
const second = invoke('claude', {}, { GSTACK_HOME: state, OPENAI_API_KEY: 'sk-test-secret-value' });
|
||||
expect(second.status).toBe(0);
|
||||
expect(second.stderr).not.toContain('NOTICE:');
|
||||
});
|
||||
|
||||
test('malformed Claude JSON cannot reach completion evaluation', () => {
|
||||
const result = invoke('codex',{}, {FAKE_MODE:'malformed'});
|
||||
expect(result.status).toBe(1);
|
||||
@@ -236,7 +332,9 @@ describe('generated outside-review dispatch', () => {
|
||||
test('autoplan retains its Codex timeout event and hang record', () => {
|
||||
const events = path.join(TMP, 'autoplan-events');
|
||||
const probe = path.join(BIN, 'gstack-codex-probe');
|
||||
fs.writeFileSync(probe, `_gstack_codex_select_model() { _GSTACK_CODEX_SEL=gpt-6-astra; }
|
||||
fs.writeFileSync(probe, `_gstack_codex_select_model() { _GSTACK_CODEX_SEL=gpt-6-astra; _GSTACK_CODEX_SANDBOX=read-only; }
|
||||
_gstack_codex_sandbox_preflight() { return 0; }
|
||||
_gstack_codex_first_use_notice() { :; }
|
||||
_gstack_codex_timeout_wrapper() { echo 'Partial finding'; return 124; }
|
||||
_gstack_codex_log_event() { printf '%s %s\\n' "$1" "$2" >> "$FAKE_EVENTS"; }
|
||||
_gstack_codex_log_hang() { printf '%s %s\\n' "$1" "$2" >> "$FAKE_EVENTS"; }
|
||||
@@ -268,6 +366,7 @@ _gstack_codex_log_hang() { printf '%s %s\\n' "$1" "$2" >> "$FAKE_EVENTS"; }
|
||||
const pidFile = path.join(TMP, 'stubborn.pid');
|
||||
fs.mkdirSync(stubborn, { recursive: true });
|
||||
fs.writeFileSync(path.join(stubborn, 'codex'), `#!/bin/bash
|
||||
[ "$1" = sandbox ] && exit 0
|
||||
trap '' TERM
|
||||
echo $$ > "$STUBBORN_PID"
|
||||
echo 'Partial finding before the deadline'
|
||||
|
||||
@@ -140,7 +140,7 @@ test.each(ALL_HOST_CONFIGS.map(({ name }) => name))('%s: ship adversarial approv
|
||||
test('outside challenge availability never resets fix limits or bypasses blocking owners', () => {
|
||||
const adversarial = compact(readTemplate('ship/sections/adversarial.md'));
|
||||
expect(adversarial).toMatch(/an unavailable outside challenge does not block shipping by itself/i);
|
||||
expect(adversarial).toMatch(/structured P1 and non-convergence gates still apply/i);
|
||||
expect(adversarial).toMatch(/structured P0\/P1 and non-convergence gates still apply/i);
|
||||
expect(adversarial).toMatch(/returning here never resets Step 9's three-cycle fix limit/i);
|
||||
const standaloneReview = readTemplate('review/sections/adversarial.md');
|
||||
expect(standaloneReview).toContain('Step 5 Fix-First');
|
||||
|
||||
@@ -1517,8 +1517,9 @@ describe('Codex skill', () => {
|
||||
expect(match).not.toBeNull();
|
||||
const resumeCommand = match![0];
|
||||
expect(resumeCommand).not.toContain(' -C ');
|
||||
expect(resumeCommand).not.toContain(' -s read-only');
|
||||
expect(resumeCommand).toContain("-c 'sandbox_mode=\"read-only\"'");
|
||||
expect(resumeCommand).not.toMatch(/ -s /);
|
||||
// resume takes the sandbox only as config; _gstack_codex_select_model sets it (read-only by default).
|
||||
expect(resumeCommand).toContain('-c "sandbox_mode=\\"${_GSTACK_CODEX_SANDBOX:?}\\""');
|
||||
});
|
||||
|
||||
test('codex union contains cost tracking', () => {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/** INV-1: every gate reason code users can see has a section in docs/troubleshooting.md. */
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { GATE_OUTCOMES, gateOutcomeLine, type GateReason } from '../lib/gate-outcomes';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const doc = fs.readFileSync(path.join(ROOT, 'docs/troubleshooting.md'), 'utf8');
|
||||
const anchors = new Set([...doc.matchAll(/<a id="([a-z0-9-]+)"><\/a>/g)].map(m => m[1]));
|
||||
|
||||
describe('docs/troubleshooting.md anchors', () => {
|
||||
for (const [code, outcome] of Object.entries(GATE_OUTCOMES)) {
|
||||
test(`${code} → #${outcome.anchor}`, () => {
|
||||
expect(anchors.has(outcome.anchor)).toBe(true);
|
||||
// The anchor introduces a section: the next line is its heading.
|
||||
const after = doc.slice(doc.indexOf(`<a id="${outcome.anchor}"></a>`)).split('\n')[1] ?? '';
|
||||
expect(after).toMatch(/^#{2,4} /);
|
||||
});
|
||||
}
|
||||
|
||||
test('anchors are unique and every outcome has a fix action', () => {
|
||||
const all = [...doc.matchAll(/<a id="([a-z0-9-]+)"><\/a>/g)].map(m => m[1]);
|
||||
expect(new Set(all).size).toBe(all.length);
|
||||
const codes = Object.values(GATE_OUTCOMES).map(o => o.anchor);
|
||||
expect(new Set(codes).size).toBe(codes.length);
|
||||
for (const outcome of Object.values(GATE_OUTCOMES)) expect(outcome.fix.trim().length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('outcome lines follow the not run / unavailable / unverified shapes', () => {
|
||||
const line = (code: GateReason, detail?: string) => gateOutcomeLine('Codex outside review', code, detail);
|
||||
expect(line('disabled')).toBe('Codex outside review: not run (outside reviews are turned off (codex_reviews=disabled)). Fix: gstack-config set codex_reviews enabled.');
|
||||
expect(line('sandbox_unavailable', 'bwrap: No permissions to create new namespace'))
|
||||
.toBe("Codex outside review unavailable: Codex's sandbox could not start here (bwrap: No permissions to create new namespace). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1.");
|
||||
expect(line('untagged_review')).toMatch(/^Codex outside review: ran, verdict unverified \(.+\)\. Fix: read the output above/);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user