gstack/bin/gstack-codex-probe

#!/usr/bin/env bash
# gstack-codex-probe: shared helper for /codex and /autoplan skills.
# Sourced from template bash blocks; never execute directly.
#
# Functions (all prefixed with _gstack_codex_ for namespace hygiene):
#   _gstack_codex_auth_probe      — multi-signal auth check (env + file)
#   _gstack_codex_model_probe     — round-trip probe of the configured model (#2477)
#   _gstack_codex_version_check   — warn on known-bad Codex CLI versions
#   _gstack_codex_timeout_wrapper — gtimeout -> timeout -> unwrapped fallback
#   _gstack_codex_log_event       — telemetry emission to ~/.gstack/analytics/
#
# Hygiene rules (enforced by test/codex-hardening.test.ts):
#   - Never set -e / set -u / trap / IFS= / PATH= in this file.
#   - All internal vars prefix with _GSTACK_CODEX_.
#   - All functions prefix with _gstack_codex_.
#   - No command execution at source time (only function defs).

# --- Auth probe -------------------------------------------------------------

_gstack_codex_auth_probe() {
  # Multi-signal: env vars OR auth file. Avoids false negatives for env-auth
  # users (CI, platform engineers) that a file-only check would reject.
  local _codex_home="${CODEX_HOME:-$HOME/.codex}"
  # Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ]
  # alone allows whitespace; pair with a whitespace strip for robustness.
  local _k1 _k2
  _k1=$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')
  _k2=$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')
  if [ -n "$_k1" ] || [ -n "$_k2" ] || [ -f "$_codex_home/auth.json" ]; then
    echo "AUTH_OK"
    return 0
  fi
  echo "AUTH_FAILED"
  return 1
}

# --- Model round-trip probe (#2477) ------------------------------------------

_gstack_codex_model_probe() {
  # Auth-exists is a weaker signal than the auth probe implies: a ChatGPT
  # account with a stale `model = "..."` pin in ~/.codex/config.toml passes
  # the auth probe, then EVERY invocation dies with an HTTP 400 ("The
  # '<model>' model is not supported when using Codex with a ChatGPT
  # account") and no guidance. A short real round trip with the configured
  # model catches model rejection, entitlement changes, and stale pins in
  # one shot (#2477).
  #
  # Contract:
  #   MODEL_OK (exit 0)              — round trip succeeded; cached 1h.
  #   MODEL_UNUSABLE (exit 1)        — deterministic model 400; hints printed.
  #     Cached 15 min: the 400 is config-driven, so re-probing every preflight
  #     charged the affected user a 30s round trip + real tokens per review
  #     section, forever. Editing config.toml (the fix) changes the cache
  #     signature and re-probes immediately; the short TTL covers server-side
  #     entitlement recovery the signature can't see.
  #   MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; FAIL-OPEN so a
  #     slow network never wedges codex mode (the per-invocation Error
  #     Handling entry still covers a later 400). Never cached.
  #
  # Only call this AFTER _gstack_codex_auth_probe passes — probing without
  # auth just measures the auth failure again.
  local _codex_home="${CODEX_HOME:-$HOME/.codex}"
  local _gstack_home="${GSTACK_HOME:-$HOME/.gstack}"
  local _cache="$_gstack_home/.codex-model-probe"
  # Cache signature: config.toml + auth.json mtimes. Editing the model pin
  # or re-logging-in invalidates the cached MODEL_OK immediately.
  local _cfg_m _auth_m _sig
  _cfg_m=$(stat -f %m "$_codex_home/config.toml" 2>/dev/null || stat -c %Y "$_codex_home/config.toml" 2>/dev/null || echo 0)
  _auth_m=$(stat -f %m "$_codex_home/auth.json" 2>/dev/null || stat -c %Y "$_codex_home/auth.json" 2>/dev/null || echo 0)
  _sig="${_cfg_m}-${_auth_m}"
  local _now
  _now=$(date +%s 2>/dev/null || echo 0)
  if [ -f "$_cache" ]; then
    local _c_line _c_status _c_ts _c_sig
    _c_line=$(head -1 "$_cache" 2>/dev/null)
    _c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1)
    _c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2)
    _c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3)
    case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac
    if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
      echo "MODEL_OK (cached)"
      return 0
    fi
    if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
      echo "MODEL_UNUSABLE (cached)"
      echo "HINT: the rejected model comes from the 'model = ' line in $_codex_home/config.toml."
      echo "HINT: check its [notice.model_migrations] table — Codex records the intended replacement there."
      return 1
    fi
  fi
  local _out _code
  _out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only "reply OK" </dev/null 2>&1)
  _code=$?
  if [ "$_code" -eq 0 ]; then
    mkdir -p "$_gstack_home" 2>/dev/null || true
    printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null || true
    echo "MODEL_OK"
    return 0
  fi
  if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*400'; then
    mkdir -p "$_gstack_home" 2>/dev/null || true
    printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null || true
    echo "MODEL_UNUSABLE"
    printf '%s\n' "$_out" | grep -i "model" | head -3
    echo "HINT: the rejected model comes from the 'model = ' line in $_codex_home/config.toml."
    echo "HINT: check its [notice.model_migrations] table — Codex records the intended replacement there."
    _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
    return 1
  fi
  # Timeout (124) or transient failure: fail-open with a warning. The probe
  # exists to catch the deterministic model 400, not to gate on network luck.
  echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — proceeding; if invocations fail with a model 400, see the codex skill's Error Handling entry."
  return 0
}

# --- Version check ----------------------------------------------------------

_gstack_codex_version_check() {
  # Warn on known-bad Codex CLI versions. Anchored regex prevents false
  # positives like 0.120.10 or 0.120.20 from matching. 0.120.2-beta still
  # matches the bad release and gets warned (it IS buggy).
  # Update this list when a new Codex CLI version regresses.
  local _ver
  _ver=$(codex --version 2>/dev/null | head -1)
  [ -z "$_ver" ] && return 0
  if echo "$_ver" | grep -Eq '(^|[^0-9.])0\.120\.(0|1|2)([^0-9.]|$)'; then
    echo "WARN: Codex CLI $_ver has known stdin deadlock bugs. Run: npm install -g @openai/codex@latest"
    _gstack_codex_log_event "codex_version_warning"
  fi
}

# --- Timeout wrapper --------------------------------------------------------

_gstack_codex_timeout_wrapper() {
  # Resolve wrapper binary: prefer gtimeout (Homebrew coreutils on macOS),
  # fall back to timeout (Linux), else a bash-native watchdog. Arguments:
  # $1 is the duration in seconds; rest is the command to run.
  local _duration="$1"
  shift
  local _to
  _to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "")
  if [ -n "$_to" ]; then
    "$_to" "$_duration" "$@"
  else
    # Stock macOS ships neither coreutils gtimeout nor timeout(1); running
    # unwrapped let a hung `codex exec` block the probe — and the calling
    # workflow — indefinitely. Emulate: background the command, TERM it at
    # the deadline, mirror timeout(1)'s exit-124 contract. The watchdog's
    # stdout is detached so an early finish never blocks a caller's $(...)
    # capture on the orphaned sleep.
    "$@" &
    local _cmd_pid=$!
    ( sleep "$_duration" && kill -TERM "$_cmd_pid" 2>/dev/null ) >/dev/null 2>&1 &
    local _watch_pid=$!
    local _rc
    wait "$_cmd_pid"
    _rc=$?
    if kill -0 "$_watch_pid" 2>/dev/null; then
      # Command finished before the deadline. Retiring the watchdog subshell
      # also defuses its pending kill (the `&& kill` lives in the subshell);
      # its detached sleep expires harmlessly.
      kill "$_watch_pid" 2>/dev/null
      wait "$_watch_pid" 2>/dev/null
    elif [ "$_rc" -ge 128 ]; then
      _rc=124  # killed by the watchdog: report timeout(1)'s code
    fi
    return "$_rc"
  fi
}

# --- Telemetry event --------------------------------------------------------

_gstack_codex_log_event() {
  # Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl.
  # Gated on $_TEL != "off" (caller sets this from gstack-config).
  # Event types: codex_timeout, codex_auth_failed, codex_cli_missing,
  #              codex_version_warning, codex_model_unusable.
  # Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt
  # content, env var values, or auth tokens.
  local _event="$1"
  local _duration="${2:-0}"
  [ "${_TEL:-off}" = "off" ] && return 0
  mkdir -p "$HOME/.gstack/analytics" 2>/dev/null || return 0
  local _ts
  _ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)
  printf '{"skill":"codex","event":"%s","duration_s":"%s","ts":"%s"}\n' \
    "$_event" "$_duration" "$_ts" \
    >> "$HOME/.gstack/analytics/skill-usage.jsonl" 2>/dev/null || true
}

# --- Learnings log on hang --------------------------------------------------

_gstack_codex_log_hang() {
  # Invoked when a codex invocation times out (exit 124). Records an
  # operational learning so future /investigate sessions surface the pattern.
  # Best-effort: errors swallowed.
  local _mode="${1:-unknown}"
  local _prompt_size="${2:-0}"
  local _log_bin="$HOME/.claude/skills/gstack/bin/gstack-learnings-log"
  [ -x "$_log_bin" ] || return 0
  local _key="codex-hang-$(date +%s 2>/dev/null || echo unknown)"
  "$_log_bin" "$(printf '{"skill":"codex","type":"operational","key":"%s","insight":"Codex timed out after 600s during [%s] invocation. Prompt size: %s. Consider splitting prompt or checking network.","confidence":8,"source":"observed","files":["codex/SKILL.md.tmpl","autoplan/SKILL.md.tmpl"]}' "$_key" "$_mode" "$_prompt_size")" \
    >/dev/null 2>&1 || true
}