gstack/bin/gstack-codex-probe

#!/usr/bin/env bash
# gstack-codex-probe: shared helper for /codex, /autoplan and every outside
# review. Run it as a command, one subcommand per probe (`gstack-codex-probe
# help`): it prints KEY: value lines on stdout, human status lines where they
# went before, and sets nothing in the caller's shell.
#
# Subcommands (each wraps one function below):
#   select-model <exec|review> [--model <id>]  CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:
#   check-auth                                 exit 0 authed, 1 not
#   show-sandbox                               CODEX_SANDBOX:
#   check-sandbox                              exit 0, or 3 with the reason on stderr
#   probe-model <exec|review> [--model <id>]   CODEX_PROBE_STATE:; exit 0, 1, 2 or 4
#   check-version                              warnings on stderr; exit 0
#   show-first-use-notice                      notice on stderr; exit 0
#   run-with-timeout <secs> <cmd...>           the command's exit code; 124 on timeout
#   log-event <event> [detail]                 silent; exit 0
#   log-hang <skill> <prompt-size>             silent; exit 0
#   help                                       usage; an unknown subcommand exits 64
#
# Sourcing (compatibility window): skills rendered before the executed form
# source this file and call the functions directly. That still works, prints
# one deprecation line on stderr, and is removed in a release on or after
# 2026-10-21 together with the zsh self-locate block below.
#
# Functions (all prefixed with _gstack_codex_ for namespace hygiene):
#   _gstack_codex_auth_probe      — multi-signal auth check (env + file)
#   _gstack_codex_select_model    — resolve + print the runtime model (#2914)
#   _gstack_codex_sandbox_mode    — read-only, or full access only for GSTACK_CODEX_NO_SANDBOX=1
#   _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux)
#   _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
#   _gstack_codex_model_probe     — round-trip probe of the selected model (#2477);
#                                   exit 4 = the account's Codex usage limit
#   _gstack_codex_version_check   — warn on known-bad Codex CLI versions
#   _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
#   _gstack_codex_log_event       — telemetry emission to ~/.gstack/analytics/
#
# Hygiene rules for the sourced form (enforced by test/codex-hardening.test.ts):
#   - Never change -e / -u / traps / IFS / PATH in the caller shell.
#   - All internal vars prefix with _GSTACK_CODEX_.
#   - All functions prefix with _gstack_codex_.
#   - No command execution at source time (only function defs, the shared
#     self-location block below, which is parameter expansion only, and the
#     deprecation line; its _gstack_helper* variables are the one exception to
#     the prefix rule and are unset once _GSTACK_CODEX_BIN is set).

# --- Self-location (bash and zsh) -------------------------------------------
# Under zsh BASH_SOURCE is empty, so every "$dir/../scripts/..." path became
# "/..." and Codex model selection failed. The shared block below sets
# _gstack_helper_dir; inside a function zsh's %x names the function, not the
# file, so it is copied once here.
_gstack_helper=gstack-codex-probe
# === gstack self-locate (shared, byte-identical in every sourced helper) ===
# Skill blocks source helpers into the host's shell: bash, or zsh on macOS
# (Claude Code's Bash tool included). Resolved once at source time, using
# parameter expansion only: BASH_SOURCE under bash, %x under zsh (inside eval
# so bash never parses it), made absolute with $PWD; else $GSTACK_ROOT/bin
# when it holds this helper. Any other case, and any other shell, fails loudly
# instead of guessing; the message and the return come before any bash- or
# zsh-only syntax, so dash and sh print it instead of a syntax error.
_gstack_helper_dir=""
_gstack_helper_shell=""
if [ -n "${BASH_VERSION:-}" ]; then
  _gstack_helper_shell=bash
  _gstack_helper_dir="${BASH_SOURCE[0]}"
elif [ -n "${ZSH_VERSION:-}" ]; then
  _gstack_helper_shell=zsh
  eval '_gstack_helper_dir="${(%):-%x}"'
fi
case "$_gstack_helper_dir" in
  /*) _gstack_helper_dir="${_gstack_helper_dir%/*}" ;;
  */*) _gstack_helper_dir="$PWD/${_gstack_helper_dir%/*}" ;;
  ?*) _gstack_helper_dir="$PWD" ;;
esac
if [ -n "$_gstack_helper_shell" ] && [ ! -f "$_gstack_helper_dir/$_gstack_helper" ]; then
  _gstack_helper_dir=""
  if [ -n "${GSTACK_ROOT:-}" ] && [ -f "$GSTACK_ROOT/bin/$_gstack_helper" ]; then
    _gstack_helper_dir="$GSTACK_ROOT/bin"
  fi
fi
if [ -z "$_gstack_helper_shell" ] || [ -z "$_gstack_helper_dir" ]; then
  _gstack_helper_error="gstack: cannot locate $_gstack_helper (shell: ${_gstack_helper_shell:-${0##*/}}). Source it from bash or zsh, or export GSTACK_ROOT=<install dir>. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location"
  printf '%s\n' "$_gstack_helper_error" >&2
  return 1 2>/dev/null || exit 1
fi
# === end gstack self-locate ===
_GSTACK_CODEX_BIN="$_gstack_helper_dir"
unset _gstack_helper _gstack_helper_dir _gstack_helper_shell

# --- Auth probe -------------------------------------------------------------

_gstack_codex_auth_probe() {
  # Multi-signal: env vars OR auth file OR a custom provider's env_key. Avoids
  # false negatives for env-auth users (CI, platform engineers, self-hosted
  # OpenAI-compatible providers) that a file-only check would reject.
  local _codex_home="${CODEX_HOME:-$HOME/.codex}"
  # Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ]
  # alone allows whitespace; pair with a whitespace strip for robustness.
  local _k1 _k2
  _k1=$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')
  _k2=$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')
  if [ -n "$_k1" ] || [ -n "$_k2" ] || [ -f "$_codex_home/auth.json" ]; then
    echo "AUTH_OK"
    return 0
  fi
  # Custom OpenAI-compatible providers (#2192): a [model_providers.<id>]
  # table names its credential variable in env_key. Only keys inside those
  # tables count, comments are ignored, and the name must be a plain shell
  # identifier before it is dereferenced.
  local _env_key _resolved _missing=""
  if [ -f "$_codex_home/config.toml" ]; then
    while IFS= read -r _env_key; do
      case "$_env_key" in ''|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac
      # Indirect read through eval, not ${!name}: zsh rejects bash's
      # indirection, and _env_key is a validated identifier by this point.
      eval "_resolved=\${$_env_key:-}"
      _resolved=$(printf '%s' "$_resolved" | tr -d '[:space:]')
      if [ -n "$_resolved" ]; then
        echo "AUTH_OK"
        return 0
      fi
      _missing="${_missing:+$_missing, }$_env_key"
    done < <(awk '/^[[:space:]]*#/ { next }
      /^[[:space:]]*\[/ { in_provider = ($0 ~ /^[[:space:]]*\[model_providers\.[^]]+\][[:space:]]*(#.*)?$/); next }
      in_provider && /^[[:space:]]*env_key[[:space:]]*=/ { v = $0; sub(/^[^=]*=[[:space:]]*/, "", v); sub(/[[:space:]]*#.*$/, "", v); gsub(/["\047[:space:]]/, "", v); print v }' "$_codex_home/config.toml" 2>/dev/null)
  fi
  echo "AUTH_FAILED"
  [ -n "$_missing" ] && echo "HINT: $_codex_home/config.toml names a custom provider key ($_missing), but it is not set in this shell. Export it, or run codex login." >&2
  return 1
}

# --- Runtime model selection (#2914) ---------------------------------------

_gstack_codex_select_model() {
  # One selection record per invocation kind, shared by the probe and the
  # dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL,
  # Codex config.toml (review_model first for native review), gstack default.
  # The TOML is read by the TypeScript resolver; bash never parses it.
  local _kind="${1:-exec}" _explicit="${2:-}" _line _rc
  case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac
  if ! command -v bun >/dev/null 2>&1; then
    echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2
    return 1
  fi
  if [ -n "$_explicit" ]; then
    _line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit")
  else
    _line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind")
  fi
  _rc=$?
  _GSTACK_CODEX_SEL="${_line%%$'\t'*}"
  _GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}"
  case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac
  if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then
    _GSTACK_CODEX_SEL=""
    _GSTACK_CODEX_SEL_SRC=""
    echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2
    return 1
  fi
  _GSTACK_CODEX_SEL_KIND="$_kind"
  echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
  _gstack_codex_sandbox_mode
}

# --- Sandbox (B1) -----------------------------------------------------------

_gstack_codex_sandbox_mode() {
  # Every gstack Codex command passes -s / sandbox_mode from this variable, so
  # one place decides it. Codex has no read-only mode without its sandbox, so
  # the escape hatch is full access: honored only for exactly 1, only from the
  # shell environment (never a dotenv file or config), and announced each use.
  if [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ]; then
    _GSTACK_CODEX_SANDBOX="danger-full-access"
    echo "WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust." >&2
  else
    _GSTACK_CODEX_SANDBOX="read-only"
  fi
}

_gstack_codex_sandbox_preflight() {
  # Free check before the first paid call: `codex sandbox` starts the same
  # bubblewrap sandbox with no model. Only a recognized sandbox failure blocks
  # (exit 3, printed as missing coverage); success, a timeout, or an older CLI
  # without the subcommand defers to the post-run classifier. Linux only, and
  # skipped when the user turned the sandbox off. Fixtures:
  # test/fixtures/codex-sandbox/.
  [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ] && return 0
  [ "$(uname -s 2>/dev/null)" = "Linux" ] || return 0
  local _out _code _line
  _out=$(_gstack_codex_timeout_wrapper 10 codex sandbox -c 'sandbox_mode="read-only"' true </dev/null 2>&1)
  _code=$?
  { [ "$_code" -eq 0 ] || [ "$_code" -eq 124 ]; } && return 0
  _line=$(printf '%s\n' "$_out" | grep -m1 -iE '^[[:space:]]*bwrap: |bubblewrap is unavailable|landlock.{0,60}(fail|error|not supported|unsupported)|seccomp.{0,60}(fail|error)|user namespaces?.{0,80}(not (allowed|permitted|supported)|denied|disabled)' | sed 's/^[[:space:]]*//' | cut -c1-240)
  [ -n "$_line" ] || return 0
  echo "CODEX_SANDBOX: unavailable"
  echo "Codex outside review unavailable: Codex's sandbox could not start here ($_line). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1." >&2
  _gstack_codex_log_event "codex_sandbox_unavailable" 2>/dev/null || true
  return 3
}

# --- First-use notice (#965) ------------------------------------------------

_gstack_codex_first_use_notice() {
  # Outside reviews are on by default and use whatever Codex login exists, so
  # the first one on a machine says where the code goes, with which account,
  # and how to turn reviews off. Shown once per state root; never blocks and
  # never prints a credential, only where it comes from.
  local _root _marker _codex_home="${CODEX_HOME:-$HOME/.codex}" _account _provider
  _root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
  [ -n "$_root" ] || return 0
  _marker="$_root/.codex-review-notice-shown"
  [ -e "$_marker" ] && return 0
  if [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in CODEX_API_KEY"
  elif [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in OPENAI_API_KEY"
  elif grep -q '"auth_mode"[[:space:]]*:[[:space:]]*"chatgpt"' "$_codex_home/auth.json" 2>/dev/null; then _account="the ChatGPT account Codex is logged in with (see: codex login status)"
  elif [ -f "$_codex_home/auth.json" ]; then _account="the API key saved by codex login in $_codex_home/auth.json"
  else _account="the credentials your Codex config.toml provider names"
  fi
  _provider=$(sed -n 's/^[[:space:]]*model_provider[[:space:]]*=[[:space:]]*"\([A-Za-z0-9_.-]*\)".*/\1/p' "$_codex_home/config.toml" 2>/dev/null | head -1)
  echo "NOTICE: gstack outside reviews send the review prompt and code to Codex (provider: ${_provider:-openai}) using $_account. Reviews stay on; this notice shows once per machine. To turn them off: gstack-config set codex_reviews disabled" >&2
  mkdir -p "$_root" 2>/dev/null && : > "$_marker" 2>/dev/null
  return 0
}

# --- Model round-trip probe (#2477) ------------------------------------------

_gstack_codex_model_probe() {
  # Auth-exists is a weaker signal than the auth probe implies: a ChatGPT
  # account can be valid while the model gstack will request is unavailable.
  # A short real round trip with the selected model catches model rejection
  # and entitlement changes in one shot (#2477). The model comes from
  # _gstack_codex_select_model ($1 = exec|review, default exec), the same
  # record the dispatch flag uses, so the probe never checks a different model.
  #
  # Contract:
  #   MODEL_OK (exit 0)              — round trip succeeded; cached 1h.
  #   MODEL_UNUSABLE (exit 1)        — deterministic model rejection (400, or a
  #     404 from a retired model or a wrong custom-provider base_url) or an
  #     invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every
  #     preflight charged the affected user a 30s round trip + real tokens per
  #     review section, forever. Changing the model or config.toml changes the
  #     cache signature and re-probes immediately; the short TTL covers
  #     server-side entitlement recovery the signature can't see.
  #   MODEL_UNUSABLE_INSTALL (exit 2) — the CLI cannot execute at all (spawn
  #     ENOENT, non-executable binary, missing vendor payload). Deterministic,
  #     so fail-open is wrong: retrying never helps. Never cached — a reinstall
  #     fixes it and must be picked up on the very next probe (#2742).
  #   MODEL_QUOTA_EXHAUSTED (exit 4) — Codex's usage-limit or insufficient_quota
  #     error on a failed, non-timeout call: the account, not the model. Codex's
  #     own line (with its reset time) is relayed verbatim; cached 15 min under
  #     the same signature as MODEL_UNUSABLE, so later fresh-shell blocks in the
  #     run make no Codex call. GSTACK_CODEX_PROBE_RETRY=1 skips both negative
  #     cache entries for that probe.
  #   MODEL_PROBE_RATE_LIMITED (exit 0) — a plain 429 / rate-limit error: often
  #     gone in seconds, so the review still runs, like the inconclusive case
  #     (_GSTACK_CODEX_PROBE_STATE=rate_limited, CODEX_MODE: unverified
  #     (rate_limited)). Codex's line is relayed verbatim; never cached.
  #   MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still
  #     runs so a slow network never wedges codex mode, but readiness is
  #     unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE:
  #     unverified) and the review's own classification decides. Never cached.
  #
  # Quota and rate-limit signatures read only Codex's own trailing error lines
  # (ERROR: / stream error), never the whole output: it echoes the prompt.
  #
  # Only call this AFTER _gstack_codex_auth_probe passes — probing without
  # auth just measures the auth failure again.
  local _kind="${1:-exec}"
  if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then
    if ! _gstack_codex_select_model "$_kind"; then
      echo "MODEL_UNUSABLE"
      echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
      return 1
    fi
  fi
  local _model="$_GSTACK_CODEX_SEL"
  local _codex_home="${CODEX_HOME:-$HOME/.codex}"
  # State root from the shared twin (sourced in a subshell, so the caller's
  # shell is untouched). A missing twin only disables the cache.
  local _gstack_home _cache=""
  _gstack_home="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || _gstack_home=""
  [ -n "$_gstack_home" ] && _cache="$_gstack_home/.codex-model-probe"
  # Cache signature (#2787): everything that can change the round trip's
  # outcome without the hour passing — the model, the Codex home and its
  # config.toml mtime, the auth mode (which env var or auth.json, plus its
  # mtime; never a credential value), and the resolved codex binary and its
  # mtime (an upgrade or PATH swap re-probes). Folded through cksum so paths
  # with spaces stay one field of the cache line.
  # GNU-first stat order + numeric validation (the #2195 pattern): on GNU
  # stat, `-f` means FILESYSTEM mode, so the BSD-first form emitted a
  # multi-line filesystem block on Linux — the signature then never matched
  # its own cache line and the cache missed on every read. BSD stat rejects
  # `-c` cleanly, so GNU-first degrades correctly on macOS.
  local _cfg_m _auth_m _auth_mode="" _bin _bin_m _sig
  _cfg_m=$(stat -c %Y "$_codex_home/config.toml" 2>/dev/null || stat -f %m "$_codex_home/config.toml" 2>/dev/null || echo 0)
  _auth_m=$(stat -c %Y "$_codex_home/auth.json" 2>/dev/null || stat -f %m "$_codex_home/auth.json" 2>/dev/null || echo 0)
  case "$_cfg_m" in ''|*[!0-9]*) _cfg_m=0 ;; esac
  case "$_auth_m" in ''|*[!0-9]*) _auth_m=0 ;; esac
  [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-codex,"
  [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-openai,"
  [ -f "$_codex_home/auth.json" ] && _auth_mode="${_auth_mode}file,"
  _bin=$(command -v codex 2>/dev/null || echo none)
  _bin=$(readlink -f "$_bin" 2>/dev/null || printf '%s' "$_bin")
  _bin_m=$(stat -c %Y "$_bin" 2>/dev/null || stat -f %m "$_bin" 2>/dev/null || echo 0)
  case "$_bin_m" in ''|*[!0-9]*) _bin_m=0 ;; esac
  _sig=$(printf '%s\n' "$_model" "$_codex_home" "$_cfg_m" "$_auth_mode" "$_auth_m" "$_bin" "$_bin_m" | cksum | tr -d ' ')
  local _now
  _now=$(date +%s 2>/dev/null || echo 0)
  local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md"
  if [ -n "$_cache" ] && [ -f "$_cache" ]; then
    local _c_line _c_status _c_ts _c_sig
    _c_line=$(head -1 "$_cache" 2>/dev/null)
    _c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1)
    _c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2)
    _c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3)
    case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac
    if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
      echo "MODEL_OK (cached)"
      return 0
    fi
    [ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry"
    if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
      echo "MODEL_QUOTA_EXHAUSTED (cached)"
      sed -n '2,4p' "$_cache" 2>/dev/null
      echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
      return 4
    fi
    if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
      echo "MODEL_UNUSABLE (cached)"
      echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
      echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
      return 1
    fi
  fi
  local _out _code
  _out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" </dev/null 2>&1)
  _code=$?
  if [ "$_code" -eq 0 ]; then
    [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
    echo "MODEL_OK"
    return 0
  fi
  # A usage limit is the account, not the model or the network: Codex says so
  # plainly ("You've hit your usage limit ... try again at <time>"), and the
  # inconclusive branch used to drop that line and re-probe in every skill.
  # Only a failed, non-timeout call counts, and only Codex's own trailing error
  # lines are read. Quota wins over rate limit: a 429 body that names
  # insufficient_quota is the quota.
  local _err _q_lines
  if [ "$_code" -ne 124 ]; then
    _err=$(printf '%s\n' "$_out" | awk '
      /^[[:space:]]*(\[[^]]*\][[:space:]]*)?(ERROR:|stream error)/ { buf = buf $0 "\n"; next }
      /^[[:space:]]*$/ { next }
      { buf = "" }
      END { printf "%s", buf }')
  fi
  _q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3)
  if [ -n "$_q_lines" ]; then
    [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_QUOTA_EXHAUSTED %s %s\n%s\n' "$_now" "$_sig" "$_q_lines" > "$_cache" 2>/dev/null; }
    echo "MODEL_QUOTA_EXHAUSTED"
    printf '%s\n' "$_q_lines"
    echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted"
    _gstack_codex_log_event "codex_quota_exhausted" 2>/dev/null || true
    return 4
  fi
  _q_lines=$(printf '%s' "$_err" | grep -iE "rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)" | head -3)
  if [ -n "$_q_lines" ]; then
    _GSTACK_CODEX_PROBE_STATE="rate_limited"
    echo "MODEL_PROBE_RATE_LIMITED — CODEX_MODE: unverified (rate_limited); proceeding, and the review's own result still decides."
    printf '%s\n' "$_q_lines"
    echo "HINT: Codex rate-limited the check (Codex's line above). Not cached; if the review itself is rate-limited, it reports missing coverage. $_docs#codex-rate-limited"
    _gstack_codex_log_event "codex_rate_limited" 2>/dev/null || true
    return 0
  fi
  # A model rejection is deterministic: a 400 ("is not supported", "requires a
  # newer version of Codex") or a 404. A retired model answers 404 "does not
  # exist or you do not have access to it", and a custom provider with a wrong
  # base_url answers a bare 404; neither is network luck, so neither may fall
  # through to the inconclusive branch below and report ready (#2843).
  if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
    [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
    echo "MODEL_UNUSABLE"
    printf '%s\n' "$_out" | grep -iE "model|404" | head -3
    echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
    if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then
      echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)."
      _gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
    elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
      echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
      echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
      _gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
    else
      echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
      _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
    fi
    return 1
  fi
  # A CLI that cannot execute is deterministic, not transient: the fail-open
  # below exists for network luck, and swallowing this here is what let a
  # missing vendor binary report CODEX_MODE: ready while every Codex pass was
  # silently skipped (#2742). 126 = found but not executable, 127 = not found.
  # String signatures only count on a FAILED, NON-TIMEOUT spawn: a successful
  # response that mentions "permission denied" must not classify as broken,
  # and neither may a timed-out (124) probe whose partial output quotes such
  # strings — 124 keeps its fail-open contract below.
  _BROKEN_SIG='ENOENT|ENOEXEC|EACCES|no such file or directory|cannot execute binary file|not executable|permission denied'
  if [ "$_code" -eq 126 ] || [ "$_code" -eq 127 ] || { [ "$_code" -ne 0 ] && [ "$_code" -ne 124 ] && printf '%s' "$_out" | grep -qiE "$_BROKEN_SIG"; }; then
    echo "MODEL_UNUSABLE_INSTALL"
    printf '%s\n' "$_out" | grep -iE "$_BROKEN_SIG" | head -3
    echo "HINT: the Codex CLI is on PATH but cannot run — its binary or vendor payload is missing."
    echo "HINT: reinstall with: npm install -g @openai/codex"
    _gstack_codex_log_event "codex_broken_install" 2>/dev/null || true
    return 2
  fi
  # Timeout (124) or transient failure: fail-open with a warning. The probe
  # exists to catch the deterministic model 400, not to gate on network luck.
  # Callers report CODEX_MODE: unverified, never ready; the exit stays 0 so
  # skills rendered before this marker keep running the review.
  _GSTACK_CODEX_PROBE_STATE="inconclusive"
  echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — CODEX_MODE: unverified; proceeding, and the review's own result is still checked. If invocations fail with a model 400, see the codex skill's Error Handling entry."
  return 0
}

# --- Version check ----------------------------------------------------------

_gstack_codex_version_check() {
  # Warn on known-bad Codex CLI versions. Anchored regex prevents false
  # positives like 0.120.10 or 0.120.20 from matching. 0.120.2-beta still
  # matches the bad release and gets warned (it IS buggy).
  # Update this list when a new Codex CLI version regresses.
  local _ver _vcode
  # Capture the code from codex, not from `head` — a pipeline reports the LAST
  # command's status, which is why a CLI that only ever printed a spawn error
  # still read as healthy here (#2742). Keep stderr: it carries the diagnosis.
  _ver=$(codex --version 2>&1)
  _vcode=$?
  _ver=$(printf '%s' "$_ver" | head -1)
  # Only a NON-ZERO exit is evidence of a broken CLI. Empty-but-successful
  # output stays silent by design (a CLI may legitimately print nothing), which
  # the "empty output → OK" case in this file's suite pins.
  if [ "$_vcode" -ne 0 ]; then
    echo "WARN: \`codex --version\` failed (exit $_vcode) — the CLI is on PATH but may not be runnable."
    [ -n "$_ver" ] && echo "WARN: it said: $_ver"
    echo "WARN: if Codex passes are being skipped, reinstall with: npm install -g @openai/codex"
    _gstack_codex_log_event "codex_version_unreadable" 2>/dev/null || true
    return 0
  fi
  [ -z "$_ver" ] && return 0
  if echo "$_ver" | grep -Eq '(^|[^0-9.])0\.120\.(0|1|2)([^0-9.]|$)'; then
    echo "WARN: Codex CLI $_ver has known stdin deadlock bugs. Run: npm install -g @openai/codex@latest"
    _gstack_codex_log_event "codex_version_warning"
  fi
}

# --- Timeout wrapper --------------------------------------------------------

_gstack_codex_timeout_wrapper() {
  # Resolve wrapper binary: prefer gtimeout (Homebrew coreutils on macOS),
  # fall back to timeout (Linux), else a bash-native watchdog. Arguments:
  # $1 is the duration in seconds; rest is the command to run.
  # A child that ignores TERM is KILLed after a grace period
  # (_GSTACK_CODEX_KILL_AFTER seconds, default 10), so the deadline always
  # ends the provider before the caller's outer tool gate (#2776). Either
  # way a deadline reports timeout(1)'s exit 124; output already written stays.
  local _duration="$1"
  shift
  local _grace="${_GSTACK_CODEX_KILL_AFTER:-10}"
  local _to _rc _start
  _to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "")
  if [ -n "$_to" ]; then
    _start=$(date +%s 2>/dev/null || echo 0)
    "$_to" -k "$_grace" "$_duration" "$@"
    _rc=$?
    # 137 = timeout(1)'s KILL escalation; it is still the deadline firing.
    if [ "$_rc" -eq 137 ] && [ $(( $(date +%s 2>/dev/null || echo 0) - _start )) -ge "${_duration%%.*}" ]; then
      _rc=124
    fi
    return "$_rc"
  else
    # Stock macOS ships neither coreutils gtimeout nor timeout(1); running
    # unwrapped let a hung `codex exec` block the probe — and the calling
    # workflow — indefinitely. Emulate: background the command, TERM it at
    # the deadline, KILL it after the grace period if it is still alive, and
    # mirror timeout(1)'s exit-124 contract. The watchdog's stdout is
    # detached so an early finish never blocks a caller's $(...) capture on
    # the orphaned sleep.
    # Without job control a background command's stdin is /dev/null; keep the
    # caller's stdin so prompts fed with `codex exec -` arrive (#1674).
    "$@" <&0 &
    local _cmd_pid=$!
    (
      sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0  # best-effort: the command already exited, nothing to stop
      _GSTACK_CODEX_WAITED=0
      while [ "$_GSTACK_CODEX_WAITED" -lt "$_grace" ] && kill -0 "$_cmd_pid" 2>/dev/null; do
        sleep 1
        _GSTACK_CODEX_WAITED=$((_GSTACK_CODEX_WAITED + 1))
      done
      # Freeze the command first: killing its children while it runs lets it
      # continue past the killed child and print output after its deadline.
      kill -STOP "$_cmd_pid" 2>/dev/null
      pkill -KILL -P "$_cmd_pid" 2>/dev/null
      kill -KILL "$_cmd_pid" 2>/dev/null
      exit 124
    ) >/dev/null 2>&1 &
    local _watch_pid=$!
    wait "$_cmd_pid"
    _rc=$?
    kill "$_watch_pid" 2>/dev/null
    wait "$_watch_pid" 2>/dev/null
    if [ "$?" -eq 124 ]; then
      _rc=124
    fi
    return "$_rc"
  fi
}

# --- Telemetry event --------------------------------------------------------

_gstack_codex_log_event() {
  # Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl.
  # Gated on $_TEL != "off" (caller sets this from gstack-config).
  # Event types: codex_timeout, codex_auth_failed, codex_cli_missing,
  #              codex_version_warning, codex_model_unusable, codex_sandbox_unavailable,
  #              codex_model_retired, codex_cli_outdated, codex_quota_exhausted,
  #              codex_rate_limited.
  # Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt
  # content, env var values, or auth tokens.
  local _event="$1"
  local _duration="${2:-0}"
  [ "${_TEL:-off}" = "off" ] && return 0
  local _root
  _root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
  [ -n "$_root" ] || return 0
  mkdir -p "$_root/analytics" 2>/dev/null || return 0
  local _ts
  _ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)
  printf '{"skill":"codex","event":"%s","duration_s":"%s","ts":"%s"}\n' \
    "$_event" "$_duration" "$_ts" \
    >> "$_root/analytics/skill-usage.jsonl" 2>/dev/null || true
}

# --- Learnings log on hang --------------------------------------------------

_gstack_codex_log_hang() {
  # Invoked when a codex invocation times out (exit 124). Records an
  # operational learning so future /investigate sessions surface the pattern.
  # Best-effort: errors swallowed.
  local _mode="${1:-unknown}"
  local _prompt_size="${2:-0}"
  local _log_bin="$HOME/.claude/skills/gstack/bin/gstack-learnings-log"
  [ -x "$_log_bin" ] || return 0
  local _key="codex-hang-$(date +%s 2>/dev/null || echo unknown)"
  "$_log_bin" "$(printf '{"skill":"codex","type":"operational","key":"%s","insight":"Codex hit its timeout wrapper during [%s] invocation. Prompt size: %s. Consider splitting prompt or checking network.","confidence":8,"source":"observed","files":["codex/SKILL.md.tmpl","autoplan/SKILL.md.tmpl"]}' "$_key" "$_mode" "$_prompt_size")" \
    >/dev/null 2>&1 || true
}

# --- Executed subcommands ----------------------------------------------------

_gstack_codex_usage() {
  printf '%s\n' \
    'Usage: gstack-codex-probe <subcommand> [args]' \
    '  select-model <exec|review> [--model <id>]  print CODEX_SEL:, CODEX_SEL_KIND: and CODEX_SANDBOX:; exit 0 or 1' \
    '  check-auth                                 exit 0 when Codex credentials exist, 1 otherwise' \
    '  show-sandbox                               print CODEX_SANDBOX: read-only (or danger-full-access)' \
    '  check-sandbox                              free sandbox check; exit 0, or 3 with the reason on stderr' \
    '  probe-model <exec|review> [--model <id>]   paid round trip; print CODEX_PROBE_STATE:; exit 0, 1, 2 or 4' \
    '  check-version                              warn on stderr about known-bad Codex CLI versions; exit 0' \
    '  show-first-use-notice                      once per machine, say where reviews go; exit 0' \
    '  run-with-timeout <secs> <cmd...>           run cmd under a deadline; its exit code, 124 on timeout' \
    '  log-event <event> [detail]                 record a telemetry event when telemetry is on; exit 0' \
    '  log-hang <skill> <prompt-size>             record a timeout learning; exit 0' \
    '  help                                       this text'
}

# Parses "<exec|review> [--model <id>]" into _GSTACK_CODEX_ARG_KIND and
# _GSTACK_CODEX_ARG_MODEL; returns 64 on anything else.
_gstack_codex_kind_args() {
  _GSTACK_CODEX_ARG_KIND="${1:-}"
  _GSTACK_CODEX_ARG_MODEL=""
  case "$_GSTACK_CODEX_ARG_KIND" in exec|review) shift ;; *) return 64 ;; esac
  if [ "$#" -eq 2 ] && [ "$1" = "--model" ] && [ -n "$2" ]; then
    _GSTACK_CODEX_ARG_MODEL="$2"
  elif [ "$#" -ne 0 ]; then
    return 64
  fi
}

_gstack_codex_main() {
  local _cmd="${1:-}" _rc
  [ "$#" -gt 0 ] && shift
  case "$_cmd" in
    log-event|probe-model|check-sandbox|check-version)
      # Telemetry is the caller's choice when it exports _TEL, else gstack-config's.
      [ -n "${_TEL+set}" ] || _TEL=$("$_GSTACK_CODEX_BIN/gstack-config" get telemetry 2>/dev/null || echo off) ;;
  esac
  case "$_cmd" in
    help|-h|--help) _gstack_codex_usage ;;
    select-model)
      _gstack_codex_kind_args "$@" || { _gstack_codex_usage >&2; return 64; }
      _gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL" || return 1
      printf 'CODEX_SEL: %s\nCODEX_SEL_KIND: %s\nCODEX_SANDBOX: %s\n' "$_GSTACK_CODEX_SEL" "$_GSTACK_CODEX_SEL_KIND" "$_GSTACK_CODEX_SANDBOX" ;;
    check-auth) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_auth_probe ;;
    show-sandbox)
      [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }
      _gstack_codex_sandbox_mode
      printf 'CODEX_SANDBOX: %s\n' "$_GSTACK_CODEX_SANDBOX" ;;
    check-sandbox) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_sandbox_preflight >&2 ;;
    probe-model)
      _gstack_codex_kind_args "$@" || { _gstack_codex_usage >&2; return 64; }
      if [ -n "$_GSTACK_CODEX_ARG_MODEL" ] && ! _gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL"; then
        echo "MODEL_UNUSABLE"
        echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
        echo "CODEX_PROBE_STATE: unusable"
        return 1
      fi
      _gstack_codex_model_probe "$_GSTACK_CODEX_ARG_KIND"
      _rc=$?
      case "$_rc" in
        0) echo "CODEX_PROBE_STATE: ${_GSTACK_CODEX_PROBE_STATE:-ok}" ;;
        1) echo "CODEX_PROBE_STATE: unusable" ;;
        2) echo "CODEX_PROBE_STATE: broken_install" ;;
        4) echo "CODEX_PROBE_STATE: quota_exhausted" ;;
        *) echo "CODEX_PROBE_STATE: error" ;;
      esac
      return "$_rc" ;;
    check-version) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_version_check >&2; return 0 ;;
    show-first-use-notice) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_first_use_notice; return 0 ;;
    run-with-timeout)
      case "${1:-}" in ''|*[!0-9.]*) _gstack_codex_usage >&2; return 64 ;; esac
      [ "$#" -ge 2 ] || { _gstack_codex_usage >&2; return 64; }
      _gstack_codex_timeout_wrapper "$@" ;;
    log-event) [ "$#" -ge 1 ] && _gstack_codex_log_event "$@" >/dev/null 2>&1; return 0 ;;
    log-hang) _gstack_codex_log_hang "$@" >/dev/null 2>&1; return 0 ;;
    *) _gstack_codex_usage >&2; return 64 ;;
  esac
}

# Executed (bash through the shebang, $0 names this file): run one subcommand.
# Sourced, including eval and zsh: keep the functions and say the form is going.
if [ -n "${BASH_VERSION:-}" ] && [ "${0##*/}" = "gstack-codex-probe" ]; then
  _gstack_codex_main "$@"
  exit $?
fi
printf '%s\n' "gstack: sourcing gstack-codex-probe is deprecated and stops working in a release on or after 2026-10-21. Run /gstack-upgrade so your skills run it as a command (gstack-codex-probe help)." >&2