#!/usr/bin/env bash # gstack-codex-probe: shared helper for /codex and /autoplan skills. # Sourced from template bash blocks; never execute directly. # # Functions (all prefixed with _gstack_codex_ for namespace hygiene): # _gstack_codex_auth_probe — multi-signal auth check (env + file) # _gstack_codex_select_model — resolve + print the runtime model (#2914) # _gstack_codex_sandbox_mode — read-only, or full access only for GSTACK_CODEX_NO_SANDBOX=1 # _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux) # _gstack_codex_model_probe — round-trip probe of the selected model (#2477) # _gstack_codex_version_check — warn on known-bad Codex CLI versions # _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog # _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/ # # Hygiene rules (enforced by test/codex-hardening.test.ts): # - Never change -e / -u / traps / IFS / PATH in the caller shell. # - All internal vars prefix with _GSTACK_CODEX_. # - All functions prefix with _gstack_codex_. # - No command execution at source time (only function defs). # --- Auth probe ------------------------------------------------------------- _gstack_codex_auth_probe() { # Multi-signal: env vars OR auth file OR a custom provider's env_key. Avoids # false negatives for env-auth users (CI, platform engineers, self-hosted # OpenAI-compatible providers) that a file-only check would reject. local _codex_home="${CODEX_HOME:-$HOME/.codex}" # Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ] # alone allows whitespace; pair with a whitespace strip for robustness. local _k1 _k2 _k1=$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]') _k2=$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]') if [ -n "$_k1" ] || [ -n "$_k2" ] || [ -f "$_codex_home/auth.json" ]; then echo "AUTH_OK" return 0 fi # Custom OpenAI-compatible providers (#2192): a [model_providers.] # table names its credential variable in env_key. Only keys inside those # tables count, comments are ignored, and the name must be a plain shell # identifier before it is dereferenced. local _env_key _resolved _missing="" if [ -f "$_codex_home/config.toml" ]; then while IFS= read -r _env_key; do case "$_env_key" in ''|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac _resolved=$(printf '%s' "${!_env_key:-}" | tr -d '[:space:]') if [ -n "$_resolved" ]; then echo "AUTH_OK" return 0 fi _missing="${_missing:+$_missing, }$_env_key" done < <(awk '/^[[:space:]]*#/ { next } /^[[:space:]]*\[/ { in_provider = ($0 ~ /^[[:space:]]*\[model_providers\.[^]]+\][[:space:]]*(#.*)?$/); next } in_provider && /^[[:space:]]*env_key[[:space:]]*=/ { v = $0; sub(/^[^=]*=[[:space:]]*/, "", v); sub(/[[:space:]]*#.*$/, "", v); gsub(/["\047[:space:]]/, "", v); print v }' "$_codex_home/config.toml" 2>/dev/null) fi echo "AUTH_FAILED" [ -n "$_missing" ] && echo "HINT: $_codex_home/config.toml names a custom provider key ($_missing), but it is not set in this shell. Export it, or run codex login." >&2 return 1 } # --- Runtime model selection (#2914) --------------------------------------- _gstack_codex_select_model() { # One selection record per invocation kind, shared by the probe and the # dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL, # Codex config.toml (review_model first for native review), gstack default. # The TOML is read by the TypeScript resolver; bash never parses it. local _kind="${1:-exec}" _explicit="${2:-}" _line _rc case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac if ! command -v bun >/dev/null 2>&1; then echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2 return 1 fi if [ -n "$_explicit" ]; then _line=$(bun run "${BASH_SOURCE[0]%/*}/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit") else _line=$(bun run "${BASH_SOURCE[0]%/*}/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind") fi _rc=$? _GSTACK_CODEX_SEL="${_line%%$'\t'*}" _GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}" case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then _GSTACK_CODEX_SEL="" _GSTACK_CODEX_SEL_SRC="" echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2 return 1 fi _GSTACK_CODEX_SEL_KIND="$_kind" echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2 _gstack_codex_sandbox_mode } # --- Sandbox (B1) ----------------------------------------------------------- _gstack_codex_sandbox_mode() { # Every gstack Codex command passes -s / sandbox_mode from this variable, so # one place decides it. Codex has no read-only mode without its sandbox, so # the escape hatch is full access: honored only for exactly 1, only from the # shell environment (never a dotenv file or config), and announced each use. if [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ]; then _GSTACK_CODEX_SANDBOX="danger-full-access" echo "WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust." >&2 else _GSTACK_CODEX_SANDBOX="read-only" fi } _gstack_codex_sandbox_preflight() { # Free check before the first paid call: `codex sandbox` starts the same # bubblewrap sandbox with no model. Only a recognized sandbox failure blocks # (exit 3, printed as missing coverage); success, a timeout, or an older CLI # without the subcommand defers to the post-run classifier. Linux only, and # skipped when the user turned the sandbox off. Fixtures: # test/fixtures/codex-sandbox/. [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ] && return 0 [ "$(uname -s 2>/dev/null)" = "Linux" ] || return 0 local _out _code _line _out=$(_gstack_codex_timeout_wrapper 10 codex sandbox -c 'sandbox_mode="read-only"' true &1) _code=$? { [ "$_code" -eq 0 ] || [ "$_code" -eq 124 ]; } && return 0 _line=$(printf '%s\n' "$_out" | grep -m1 -iE '^[[:space:]]*bwrap: |bubblewrap is unavailable|landlock.{0,60}(fail|error|not supported|unsupported)|seccomp.{0,60}(fail|error)|user namespaces?.{0,80}(not (allowed|permitted|supported)|denied|disabled)' | sed 's/^[[:space:]]*//' | cut -c1-240) [ -n "$_line" ] || return 0 echo "CODEX_SANDBOX: unavailable" echo "Codex outside review unavailable: Codex's sandbox could not start here ($_line). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1." >&2 _gstack_codex_log_event "codex_sandbox_unavailable" 2>/dev/null || true return 3 } # --- Model round-trip probe (#2477) ------------------------------------------ _gstack_codex_model_probe() { # Auth-exists is a weaker signal than the auth probe implies: a ChatGPT # account can be valid while the model gstack will request is unavailable. # A short real round trip with the selected model catches model rejection # and entitlement changes in one shot (#2477). The model comes from # _gstack_codex_select_model ($1 = exec|review, default exec), the same # record the dispatch flag uses, so the probe never checks a different model. # # Contract: # MODEL_OK (exit 0) — round trip succeeded; cached 1h. # MODEL_UNUSABLE (exit 1) — deterministic model rejection (400, or a # 404 from a retired model or a wrong custom-provider base_url) or an # invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every # preflight charged the affected user a 30s round trip + real tokens per # review section, forever. Changing the model or config.toml changes the # cache signature and re-probes immediately; the short TTL covers # server-side entitlement recovery the signature can't see. # MODEL_UNUSABLE_INSTALL (exit 2) — the CLI cannot execute at all (spawn # ENOENT, non-executable binary, missing vendor payload). Deterministic, # so fail-open is wrong: retrying never helps. Never cached — a reinstall # fixes it and must be picked up on the very next probe (#2742). # MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still # runs so a slow network never wedges codex mode, but readiness is # unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE: # unverified) and the review's own classification decides. Never cached. # # Only call this AFTER _gstack_codex_auth_probe passes — probing without # auth just measures the auth failure again. local _kind="${1:-exec}" if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then if ! _gstack_codex_select_model "$_kind"; then echo "MODEL_UNUSABLE" echo "HINT: the requested Codex model is not a valid model id; fix it as described above." return 1 fi fi local _model="$_GSTACK_CODEX_SEL" local _codex_home="${CODEX_HOME:-$HOME/.codex}" # State root from the shared twin (sourced in a subshell, so the caller's # shell is untouched). A missing twin only disables the cache. local _gstack_home _cache="" _gstack_home="$(. "${BASH_SOURCE[0]%/*}/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || _gstack_home="" [ -n "$_gstack_home" ] && _cache="$_gstack_home/.codex-model-probe" # Cache signature (#2787): everything that can change the round trip's # outcome without the hour passing — the model, the Codex home and its # config.toml mtime, the auth mode (which env var or auth.json, plus its # mtime; never a credential value), and the resolved codex binary and its # mtime (an upgrade or PATH swap re-probes). Folded through cksum so paths # with spaces stay one field of the cache line. # GNU-first stat order + numeric validation (the #2195 pattern): on GNU # stat, `-f` means FILESYSTEM mode, so the BSD-first form emitted a # multi-line filesystem block on Linux — the signature then never matched # its own cache line and the cache missed on every read. BSD stat rejects # `-c` cleanly, so GNU-first degrades correctly on macOS. local _cfg_m _auth_m _auth_mode="" _bin _bin_m _sig _cfg_m=$(stat -c %Y "$_codex_home/config.toml" 2>/dev/null || stat -f %m "$_codex_home/config.toml" 2>/dev/null || echo 0) _auth_m=$(stat -c %Y "$_codex_home/auth.json" 2>/dev/null || stat -f %m "$_codex_home/auth.json" 2>/dev/null || echo 0) case "$_cfg_m" in ''|*[!0-9]*) _cfg_m=0 ;; esac case "$_auth_m" in ''|*[!0-9]*) _auth_m=0 ;; esac [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-codex," [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-openai," [ -f "$_codex_home/auth.json" ] && _auth_mode="${_auth_mode}file," _bin=$(command -v codex 2>/dev/null || echo none) _bin=$(readlink -f "$_bin" 2>/dev/null || printf '%s' "$_bin") _bin_m=$(stat -c %Y "$_bin" 2>/dev/null || stat -f %m "$_bin" 2>/dev/null || echo 0) case "$_bin_m" in ''|*[!0-9]*) _bin_m=0 ;; esac _sig=$(printf '%s\n' "$_model" "$_codex_home" "$_cfg_m" "$_auth_mode" "$_auth_m" "$_bin" "$_bin_m" | cksum | tr -d ' ') local _now _now=$(date +%s 2>/dev/null || echo 0) if [ -n "$_cache" ] && [ -f "$_cache" ]; then local _c_line _c_status _c_ts _c_sig _c_line=$(head -1 "$_cache" 2>/dev/null) _c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1) _c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2) _c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3) case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then echo "MODEL_OK (cached)" return 0 fi if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then echo "MODEL_UNUSABLE (cached)" echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." return 1 fi fi local _out _code _out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" &1) _code=$? if [ "$_code" -eq 0 ]; then [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; } echo "MODEL_OK" return 0 fi # A model rejection is deterministic: a 400 ("is not supported", "requires a # newer version of Codex") or a 404. A retired model answers 404 "does not # exist or you do not have access to it", and a custom provider with a wrong # base_url answers a bare 404; neither is network luck, so neither may fall # through to the inconclusive branch below and report ready (#2843). if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; } echo "MODEL_UNUSABLE" printf '%s\n' "$_out" | grep -iE "model|404" | head -3 echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)." _gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong." echo "HINT: choose a current model (GSTACK_CODEX_MODEL= or model in config.toml), or fix that provider's base_url." _gstack_codex_log_event "codex_model_retired" 2>/dev/null || true else echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true fi return 1 fi # A CLI that cannot execute is deterministic, not transient: the fail-open # below exists for network luck, and swallowing this here is what let a # missing vendor binary report CODEX_MODE: ready while every Codex pass was # silently skipped (#2742). 126 = found but not executable, 127 = not found. # String signatures only count on a FAILED, NON-TIMEOUT spawn: a successful # response that mentions "permission denied" must not classify as broken, # and neither may a timed-out (124) probe whose partial output quotes such # strings — 124 keeps its fail-open contract below. _BROKEN_SIG='ENOENT|ENOEXEC|EACCES|no such file or directory|cannot execute binary file|not executable|permission denied' if [ "$_code" -eq 126 ] || [ "$_code" -eq 127 ] || { [ "$_code" -ne 0 ] && [ "$_code" -ne 124 ] && printf '%s' "$_out" | grep -qiE "$_BROKEN_SIG"; }; then echo "MODEL_UNUSABLE_INSTALL" printf '%s\n' "$_out" | grep -iE "$_BROKEN_SIG" | head -3 echo "HINT: the Codex CLI is on PATH but cannot run — its binary or vendor payload is missing." echo "HINT: reinstall with: npm install -g @openai/codex" _gstack_codex_log_event "codex_broken_install" 2>/dev/null || true return 2 fi # Timeout (124) or transient failure: fail-open with a warning. The probe # exists to catch the deterministic model 400, not to gate on network luck. # Callers report CODEX_MODE: unverified, never ready; the exit stays 0 so # skills rendered before this marker keep running the review. _GSTACK_CODEX_PROBE_STATE="inconclusive" echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — CODEX_MODE: unverified; proceeding, and the review's own result is still checked. If invocations fail with a model 400, see the codex skill's Error Handling entry." return 0 } # --- Version check ---------------------------------------------------------- _gstack_codex_version_check() { # Warn on known-bad Codex CLI versions. Anchored regex prevents false # positives like 0.120.10 or 0.120.20 from matching. 0.120.2-beta still # matches the bad release and gets warned (it IS buggy). # Update this list when a new Codex CLI version regresses. local _ver _vcode # Capture the code from codex, not from `head` — a pipeline reports the LAST # command's status, which is why a CLI that only ever printed a spawn error # still read as healthy here (#2742). Keep stderr: it carries the diagnosis. _ver=$(codex --version 2>&1) _vcode=$? _ver=$(printf '%s' "$_ver" | head -1) # Only a NON-ZERO exit is evidence of a broken CLI. Empty-but-successful # output stays silent by design (a CLI may legitimately print nothing), which # the "empty output → OK" case in this file's suite pins. if [ "$_vcode" -ne 0 ]; then echo "WARN: \`codex --version\` failed (exit $_vcode) — the CLI is on PATH but may not be runnable." [ -n "$_ver" ] && echo "WARN: it said: $_ver" echo "WARN: if Codex passes are being skipped, reinstall with: npm install -g @openai/codex" _gstack_codex_log_event "codex_version_unreadable" 2>/dev/null || true return 0 fi [ -z "$_ver" ] && return 0 if echo "$_ver" | grep -Eq '(^|[^0-9.])0\.120\.(0|1|2)([^0-9.]|$)'; then echo "WARN: Codex CLI $_ver has known stdin deadlock bugs. Run: npm install -g @openai/codex@latest" _gstack_codex_log_event "codex_version_warning" fi } # --- Timeout wrapper -------------------------------------------------------- _gstack_codex_timeout_wrapper() { # Resolve wrapper binary: prefer gtimeout (Homebrew coreutils on macOS), # fall back to timeout (Linux), else a bash-native watchdog. Arguments: # $1 is the duration in seconds; rest is the command to run. # A child that ignores TERM is KILLed after a grace period # (_GSTACK_CODEX_KILL_AFTER seconds, default 10), so the deadline always # ends the provider before the caller's outer tool gate (#2776). Either # way a deadline reports timeout(1)'s exit 124; output already written stays. local _duration="$1" shift local _grace="${_GSTACK_CODEX_KILL_AFTER:-10}" local _to _rc _start _to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "") if [ -n "$_to" ]; then _start=$(date +%s 2>/dev/null || echo 0) "$_to" -k "$_grace" "$_duration" "$@" _rc=$? # 137 = timeout(1)'s KILL escalation; it is still the deadline firing. if [ "$_rc" -eq 137 ] && [ $(( $(date +%s 2>/dev/null || echo 0) - _start )) -ge "${_duration%%.*}" ]; then _rc=124 fi return "$_rc" else # Stock macOS ships neither coreutils gtimeout nor timeout(1); running # unwrapped let a hung `codex exec` block the probe — and the calling # workflow — indefinitely. Emulate: background the command, TERM it at # the deadline, KILL it after the grace period if it is still alive, and # mirror timeout(1)'s exit-124 contract. The watchdog's stdout is # detached so an early finish never blocks a caller's $(...) capture on # the orphaned sleep. # Without job control a background command's stdin is /dev/null; keep the # caller's stdin so prompts fed with `codex exec -` arrive (#1674). "$@" <&0 & local _cmd_pid=$! ( sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0 # best-effort: the command already exited, nothing to stop _GSTACK_CODEX_WAITED=0 while [ "$_GSTACK_CODEX_WAITED" -lt "$_grace" ] && kill -0 "$_cmd_pid" 2>/dev/null; do sleep 1 _GSTACK_CODEX_WAITED=$((_GSTACK_CODEX_WAITED + 1)) done pkill -KILL -P "$_cmd_pid" 2>/dev/null kill -KILL "$_cmd_pid" 2>/dev/null exit 124 ) >/dev/null 2>&1 & local _watch_pid=$! wait "$_cmd_pid" _rc=$? kill "$_watch_pid" 2>/dev/null wait "$_watch_pid" 2>/dev/null if [ "$?" -eq 124 ]; then _rc=124 fi return "$_rc" fi } # --- Telemetry event -------------------------------------------------------- _gstack_codex_log_event() { # Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl. # Gated on $_TEL != "off" (caller sets this from gstack-config). # Event types: codex_timeout, codex_auth_failed, codex_cli_missing, # codex_version_warning, codex_model_unusable, codex_sandbox_unavailable, # codex_model_retired, codex_cli_outdated. # Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt # content, env var values, or auth tokens. local _event="$1" local _duration="${2:-0}" [ "${_TEL:-off}" = "off" ] && return 0 local _root _root="$(. "${BASH_SOURCE[0]%/*}/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0 [ -n "$_root" ] || return 0 mkdir -p "$_root/analytics" 2>/dev/null || return 0 local _ts _ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown) printf '{"skill":"codex","event":"%s","duration_s":"%s","ts":"%s"}\n' \ "$_event" "$_duration" "$_ts" \ >> "$_root/analytics/skill-usage.jsonl" 2>/dev/null || true } # --- Learnings log on hang -------------------------------------------------- _gstack_codex_log_hang() { # Invoked when a codex invocation times out (exit 124). Records an # operational learning so future /investigate sessions surface the pattern. # Best-effort: errors swallowed. local _mode="${1:-unknown}" local _prompt_size="${2:-0}" local _log_bin="$HOME/.claude/skills/gstack/bin/gstack-learnings-log" [ -x "$_log_bin" ] || return 0 local _key="codex-hang-$(date +%s 2>/dev/null || echo unknown)" "$_log_bin" "$(printf '{"skill":"codex","type":"operational","key":"%s","insight":"Codex hit its timeout wrapper during [%s] invocation. Prompt size: %s. Consider splitting prompt or checking network.","confidence":8,"source":"observed","files":["codex/SKILL.md.tmpl","autoplan/SKILL.md.tmpl"]}' "$_key" "$_mode" "$_prompt_size")" \ >/dev/null 2>&1 || true }