mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
garrytanandvenkatareddyg
3542322819
The probe's quota signature used to treat any rate limit, "too many requests" or 429 as quota, and read the whole output, which echoes the prompt. Now only Codex's trailing ERROR: / stream error lines are read. A usage-limit or insufficient_quota line is MODEL_QUOTA_EXHAUSTED (exit 4, cached 15 minutes; quota wins over a 429 that names insufficient_quota); a plain 429 is MODEL_PROBE_RATE_LIMITED (exit 0, state rate_limited, never cached), reported as CODEX_MODE: unverified (rate_limited) so the review still runs and decides. The quota HINT says how many minutes gstack skips Codex and how to retry now (GSTACK_CODEX_PROBE_RETRY=1 skips the negative cache for that probe, or delete <state root>/.codex-model-probe), with the full troubleshooting URL. classifyOutsideReview reads the same trailing lines: a mid-review quota is unavailable (quota_exhausted), a mid-review 429 unavailable (rate_limited). New gate row rate_limited and docs anchor codex-rate-limited; the quota row documents the retry. Tests cover each line shape, the echoed prompt, exit 0, timeout, retry recovery, probe/classifier parity, and two fresh-shell preflight blocks where the second makes no paid Codex call. Co-authored-by: venkatareddyg <venkatareddyg@users.noreply.github.com>
535 lines
31 KiB
Bash
Executable File
535 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# gstack-codex-probe: shared helper for /codex and /autoplan skills.
|
|
# Sourced from template bash blocks; never execute directly.
|
|
#
|
|
# Functions (all prefixed with _gstack_codex_ for namespace hygiene):
|
|
# _gstack_codex_auth_probe — multi-signal auth check (env + file)
|
|
# _gstack_codex_select_model — resolve + print the runtime model (#2914)
|
|
# _gstack_codex_sandbox_mode — read-only, or full access only for GSTACK_CODEX_NO_SANDBOX=1
|
|
# _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux)
|
|
# _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
|
|
# _gstack_codex_model_probe — round-trip probe of the selected model (#2477);
|
|
# exit 4 = the account's Codex usage limit
|
|
# _gstack_codex_version_check — warn on known-bad Codex CLI versions
|
|
# _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
|
|
# _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/
|
|
#
|
|
# Hygiene rules (enforced by test/codex-hardening.test.ts):
|
|
# - Never change -e / -u / traps / IFS / PATH in the caller shell.
|
|
# - All internal vars prefix with _GSTACK_CODEX_.
|
|
# - All functions prefix with _gstack_codex_.
|
|
# - No command execution at source time (only function defs and the shared
|
|
# self-location block below, which is parameter expansion only; its
|
|
# _gstack_helper* variables are the one exception to the prefix rule and
|
|
# are unset once _GSTACK_CODEX_BIN is set).
|
|
|
|
# --- Self-location (bash and zsh) -------------------------------------------
|
|
# Under zsh BASH_SOURCE is empty, so every "$dir/../scripts/..." path became
|
|
# "/..." and Codex model selection failed. The shared block below sets
|
|
# _gstack_helper_dir; inside a function zsh's %x names the function, not the
|
|
# file, so it is copied once here.
|
|
_gstack_helper=gstack-codex-probe
|
|
# >>> gstack self-locate (shared, byte-identical in every sourced helper) >>>
|
|
# Skill blocks source helpers into the host's shell: bash, or zsh on macOS
|
|
# (Claude Code's Bash tool included). Resolved once at source time, using
|
|
# parameter expansion only: BASH_SOURCE under bash, %x under zsh (inside eval
|
|
# so bash never parses it), made absolute with $PWD; else $GSTACK_ROOT/bin
|
|
# when it holds this helper. Any other case, and any other shell, fails loudly
|
|
# instead of guessing; the message and the return come before any bash- or
|
|
# zsh-only syntax, so dash and sh print it instead of a syntax error.
|
|
_gstack_helper_dir=""
|
|
_gstack_helper_shell=""
|
|
if [ -n "${BASH_VERSION:-}" ]; then
|
|
_gstack_helper_shell=bash
|
|
_gstack_helper_dir="${BASH_SOURCE[0]}"
|
|
elif [ -n "${ZSH_VERSION:-}" ]; then
|
|
_gstack_helper_shell=zsh
|
|
eval '_gstack_helper_dir="${(%):-%x}"'
|
|
fi
|
|
case "$_gstack_helper_dir" in
|
|
/*) _gstack_helper_dir="${_gstack_helper_dir%/*}" ;;
|
|
*/*) _gstack_helper_dir="$PWD/${_gstack_helper_dir%/*}" ;;
|
|
?*) _gstack_helper_dir="$PWD" ;;
|
|
esac
|
|
if [ -n "$_gstack_helper_shell" ] && [ ! -f "$_gstack_helper_dir/$_gstack_helper" ]; then
|
|
_gstack_helper_dir=""
|
|
if [ -n "${GSTACK_ROOT:-}" ] && [ -f "$GSTACK_ROOT/bin/$_gstack_helper" ]; then
|
|
_gstack_helper_dir="$GSTACK_ROOT/bin"
|
|
fi
|
|
fi
|
|
if [ -z "$_gstack_helper_shell" ] || [ -z "$_gstack_helper_dir" ]; then
|
|
_gstack_helper_error="gstack: cannot locate $_gstack_helper (shell: ${_gstack_helper_shell:-${0##*/}}). Source it from bash or zsh, or export GSTACK_ROOT=<install dir>. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location"
|
|
printf '%s\n' "$_gstack_helper_error" >&2
|
|
return 1 2>/dev/null || exit 1
|
|
fi
|
|
# <<< gstack self-locate <<<
|
|
_GSTACK_CODEX_BIN="$_gstack_helper_dir"
|
|
unset _gstack_helper _gstack_helper_dir _gstack_helper_shell
|
|
|
|
# --- Auth probe -------------------------------------------------------------
|
|
|
|
_gstack_codex_auth_probe() {
|
|
# Multi-signal: env vars OR auth file OR a custom provider's env_key. Avoids
|
|
# false negatives for env-auth users (CI, platform engineers, self-hosted
|
|
# OpenAI-compatible providers) that a file-only check would reject.
|
|
local _codex_home="${CODEX_HOME:-$HOME/.codex}"
|
|
# Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ]
|
|
# alone allows whitespace; pair with a whitespace strip for robustness.
|
|
local _k1 _k2
|
|
_k1=$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')
|
|
_k2=$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')
|
|
if [ -n "$_k1" ] || [ -n "$_k2" ] || [ -f "$_codex_home/auth.json" ]; then
|
|
echo "AUTH_OK"
|
|
return 0
|
|
fi
|
|
# Custom OpenAI-compatible providers (#2192): a [model_providers.<id>]
|
|
# table names its credential variable in env_key. Only keys inside those
|
|
# tables count, comments are ignored, and the name must be a plain shell
|
|
# identifier before it is dereferenced.
|
|
local _env_key _resolved _missing=""
|
|
if [ -f "$_codex_home/config.toml" ]; then
|
|
while IFS= read -r _env_key; do
|
|
case "$_env_key" in ''|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac
|
|
# Indirect read through eval, not ${!name}: zsh rejects bash's
|
|
# indirection, and _env_key is a validated identifier by this point.
|
|
eval "_resolved=\${$_env_key:-}"
|
|
_resolved=$(printf '%s' "$_resolved" | tr -d '[:space:]')
|
|
if [ -n "$_resolved" ]; then
|
|
echo "AUTH_OK"
|
|
return 0
|
|
fi
|
|
_missing="${_missing:+$_missing, }$_env_key"
|
|
done < <(awk '/^[[:space:]]*#/ { next }
|
|
/^[[:space:]]*\[/ { in_provider = ($0 ~ /^[[:space:]]*\[model_providers\.[^]]+\][[:space:]]*(#.*)?$/); next }
|
|
in_provider && /^[[:space:]]*env_key[[:space:]]*=/ { v = $0; sub(/^[^=]*=[[:space:]]*/, "", v); sub(/[[:space:]]*#.*$/, "", v); gsub(/["\047[:space:]]/, "", v); print v }' "$_codex_home/config.toml" 2>/dev/null)
|
|
fi
|
|
echo "AUTH_FAILED"
|
|
[ -n "$_missing" ] && echo "HINT: $_codex_home/config.toml names a custom provider key ($_missing), but it is not set in this shell. Export it, or run codex login." >&2
|
|
return 1
|
|
}
|
|
|
|
# --- Runtime model selection (#2914) ---------------------------------------
|
|
|
|
_gstack_codex_select_model() {
|
|
# One selection record per invocation kind, shared by the probe and the
|
|
# dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL,
|
|
# Codex config.toml (review_model first for native review), gstack default.
|
|
# The TOML is read by the TypeScript resolver; bash never parses it.
|
|
local _kind="${1:-exec}" _explicit="${2:-}" _line _rc
|
|
case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac
|
|
if ! command -v bun >/dev/null 2>&1; then
|
|
echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2
|
|
return 1
|
|
fi
|
|
if [ -n "$_explicit" ]; then
|
|
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit")
|
|
else
|
|
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind")
|
|
fi
|
|
_rc=$?
|
|
_GSTACK_CODEX_SEL="${_line%%$'\t'*}"
|
|
_GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}"
|
|
case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac
|
|
if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then
|
|
_GSTACK_CODEX_SEL=""
|
|
_GSTACK_CODEX_SEL_SRC=""
|
|
echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2
|
|
return 1
|
|
fi
|
|
_GSTACK_CODEX_SEL_KIND="$_kind"
|
|
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
|
_gstack_codex_sandbox_mode
|
|
}
|
|
|
|
# --- Sandbox (B1) -----------------------------------------------------------
|
|
|
|
_gstack_codex_sandbox_mode() {
|
|
# Every gstack Codex command passes -s / sandbox_mode from this variable, so
|
|
# one place decides it. Codex has no read-only mode without its sandbox, so
|
|
# the escape hatch is full access: honored only for exactly 1, only from the
|
|
# shell environment (never a dotenv file or config), and announced each use.
|
|
if [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ]; then
|
|
_GSTACK_CODEX_SANDBOX="danger-full-access"
|
|
echo "WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust." >&2
|
|
else
|
|
_GSTACK_CODEX_SANDBOX="read-only"
|
|
fi
|
|
}
|
|
|
|
_gstack_codex_sandbox_preflight() {
|
|
# Free check before the first paid call: `codex sandbox` starts the same
|
|
# bubblewrap sandbox with no model. Only a recognized sandbox failure blocks
|
|
# (exit 3, printed as missing coverage); success, a timeout, or an older CLI
|
|
# without the subcommand defers to the post-run classifier. Linux only, and
|
|
# skipped when the user turned the sandbox off. Fixtures:
|
|
# test/fixtures/codex-sandbox/.
|
|
[ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ] && return 0
|
|
[ "$(uname -s 2>/dev/null)" = "Linux" ] || return 0
|
|
local _out _code _line
|
|
_out=$(_gstack_codex_timeout_wrapper 10 codex sandbox -c 'sandbox_mode="read-only"' true </dev/null 2>&1)
|
|
_code=$?
|
|
{ [ "$_code" -eq 0 ] || [ "$_code" -eq 124 ]; } && return 0
|
|
_line=$(printf '%s\n' "$_out" | grep -m1 -iE '^[[:space:]]*bwrap: |bubblewrap is unavailable|landlock.{0,60}(fail|error|not supported|unsupported)|seccomp.{0,60}(fail|error)|user namespaces?.{0,80}(not (allowed|permitted|supported)|denied|disabled)' | sed 's/^[[:space:]]*//' | cut -c1-240)
|
|
[ -n "$_line" ] || return 0
|
|
echo "CODEX_SANDBOX: unavailable"
|
|
echo "Codex outside review unavailable: Codex's sandbox could not start here ($_line). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1." >&2
|
|
_gstack_codex_log_event "codex_sandbox_unavailable" 2>/dev/null || true
|
|
return 3
|
|
}
|
|
|
|
# --- First-use notice (#965) ------------------------------------------------
|
|
|
|
_gstack_codex_first_use_notice() {
|
|
# Outside reviews are on by default and use whatever Codex login exists, so
|
|
# the first one on a machine says where the code goes, with which account,
|
|
# and how to turn reviews off. Shown once per state root; never blocks and
|
|
# never prints a credential, only where it comes from.
|
|
local _root _marker _codex_home="${CODEX_HOME:-$HOME/.codex}" _account _provider
|
|
_root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
|
|
[ -n "$_root" ] || return 0
|
|
_marker="$_root/.codex-review-notice-shown"
|
|
[ -e "$_marker" ] && return 0
|
|
if [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in CODEX_API_KEY"
|
|
elif [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in OPENAI_API_KEY"
|
|
elif grep -q '"auth_mode"[[:space:]]*:[[:space:]]*"chatgpt"' "$_codex_home/auth.json" 2>/dev/null; then _account="the ChatGPT account Codex is logged in with (see: codex login status)"
|
|
elif [ -f "$_codex_home/auth.json" ]; then _account="the API key saved by codex login in $_codex_home/auth.json"
|
|
else _account="the credentials your Codex config.toml provider names"
|
|
fi
|
|
_provider=$(sed -n 's/^[[:space:]]*model_provider[[:space:]]*=[[:space:]]*"\([A-Za-z0-9_.-]*\)".*/\1/p' "$_codex_home/config.toml" 2>/dev/null | head -1)
|
|
echo "NOTICE: gstack outside reviews send the review prompt and code to Codex (provider: ${_provider:-openai}) using $_account. Reviews stay on; this notice shows once per machine. To turn them off: gstack-config set codex_reviews disabled" >&2
|
|
mkdir -p "$_root" 2>/dev/null && : > "$_marker" 2>/dev/null
|
|
return 0
|
|
}
|
|
|
|
# --- Model round-trip probe (#2477) ------------------------------------------
|
|
|
|
_gstack_codex_model_probe() {
|
|
# Auth-exists is a weaker signal than the auth probe implies: a ChatGPT
|
|
# account can be valid while the model gstack will request is unavailable.
|
|
# A short real round trip with the selected model catches model rejection
|
|
# and entitlement changes in one shot (#2477). The model comes from
|
|
# _gstack_codex_select_model ($1 = exec|review, default exec), the same
|
|
# record the dispatch flag uses, so the probe never checks a different model.
|
|
#
|
|
# Contract:
|
|
# MODEL_OK (exit 0) — round trip succeeded; cached 1h.
|
|
# MODEL_UNUSABLE (exit 1) — deterministic model rejection (400, or a
|
|
# 404 from a retired model or a wrong custom-provider base_url) or an
|
|
# invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every
|
|
# preflight charged the affected user a 30s round trip + real tokens per
|
|
# review section, forever. Changing the model or config.toml changes the
|
|
# cache signature and re-probes immediately; the short TTL covers
|
|
# server-side entitlement recovery the signature can't see.
|
|
# MODEL_UNUSABLE_INSTALL (exit 2) — the CLI cannot execute at all (spawn
|
|
# ENOENT, non-executable binary, missing vendor payload). Deterministic,
|
|
# so fail-open is wrong: retrying never helps. Never cached — a reinstall
|
|
# fixes it and must be picked up on the very next probe (#2742).
|
|
# MODEL_QUOTA_EXHAUSTED (exit 4) — Codex's usage-limit or insufficient_quota
|
|
# error on a failed, non-timeout call: the account, not the model. Codex's
|
|
# own line (with its reset time) is relayed verbatim; cached 15 min under
|
|
# the same signature as MODEL_UNUSABLE, so later fresh-shell blocks in the
|
|
# run make no Codex call. GSTACK_CODEX_PROBE_RETRY=1 skips both negative
|
|
# cache entries for that probe.
|
|
# MODEL_PROBE_RATE_LIMITED (exit 0) — a plain 429 / rate-limit error: often
|
|
# gone in seconds, so the review still runs, like the inconclusive case
|
|
# (_GSTACK_CODEX_PROBE_STATE=rate_limited, CODEX_MODE: unverified
|
|
# (rate_limited)). Codex's line is relayed verbatim; never cached.
|
|
# MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still
|
|
# runs so a slow network never wedges codex mode, but readiness is
|
|
# unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE:
|
|
# unverified) and the review's own classification decides. Never cached.
|
|
#
|
|
# Quota and rate-limit signatures read only Codex's own trailing error lines
|
|
# (ERROR: / stream error), never the whole output: it echoes the prompt.
|
|
#
|
|
# Only call this AFTER _gstack_codex_auth_probe passes — probing without
|
|
# auth just measures the auth failure again.
|
|
local _kind="${1:-exec}"
|
|
if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then
|
|
if ! _gstack_codex_select_model "$_kind"; then
|
|
echo "MODEL_UNUSABLE"
|
|
echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
|
|
return 1
|
|
fi
|
|
fi
|
|
local _model="$_GSTACK_CODEX_SEL"
|
|
local _codex_home="${CODEX_HOME:-$HOME/.codex}"
|
|
# State root from the shared twin (sourced in a subshell, so the caller's
|
|
# shell is untouched). A missing twin only disables the cache.
|
|
local _gstack_home _cache=""
|
|
_gstack_home="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || _gstack_home=""
|
|
[ -n "$_gstack_home" ] && _cache="$_gstack_home/.codex-model-probe"
|
|
# Cache signature (#2787): everything that can change the round trip's
|
|
# outcome without the hour passing — the model, the Codex home and its
|
|
# config.toml mtime, the auth mode (which env var or auth.json, plus its
|
|
# mtime; never a credential value), and the resolved codex binary and its
|
|
# mtime (an upgrade or PATH swap re-probes). Folded through cksum so paths
|
|
# with spaces stay one field of the cache line.
|
|
# GNU-first stat order + numeric validation (the #2195 pattern): on GNU
|
|
# stat, `-f` means FILESYSTEM mode, so the BSD-first form emitted a
|
|
# multi-line filesystem block on Linux — the signature then never matched
|
|
# its own cache line and the cache missed on every read. BSD stat rejects
|
|
# `-c` cleanly, so GNU-first degrades correctly on macOS.
|
|
local _cfg_m _auth_m _auth_mode="" _bin _bin_m _sig
|
|
_cfg_m=$(stat -c %Y "$_codex_home/config.toml" 2>/dev/null || stat -f %m "$_codex_home/config.toml" 2>/dev/null || echo 0)
|
|
_auth_m=$(stat -c %Y "$_codex_home/auth.json" 2>/dev/null || stat -f %m "$_codex_home/auth.json" 2>/dev/null || echo 0)
|
|
case "$_cfg_m" in ''|*[!0-9]*) _cfg_m=0 ;; esac
|
|
case "$_auth_m" in ''|*[!0-9]*) _auth_m=0 ;; esac
|
|
[ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-codex,"
|
|
[ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-openai,"
|
|
[ -f "$_codex_home/auth.json" ] && _auth_mode="${_auth_mode}file,"
|
|
_bin=$(command -v codex 2>/dev/null || echo none)
|
|
_bin=$(readlink -f "$_bin" 2>/dev/null || printf '%s' "$_bin")
|
|
_bin_m=$(stat -c %Y "$_bin" 2>/dev/null || stat -f %m "$_bin" 2>/dev/null || echo 0)
|
|
case "$_bin_m" in ''|*[!0-9]*) _bin_m=0 ;; esac
|
|
_sig=$(printf '%s\n' "$_model" "$_codex_home" "$_cfg_m" "$_auth_mode" "$_auth_m" "$_bin" "$_bin_m" | cksum | tr -d ' ')
|
|
local _now
|
|
_now=$(date +%s 2>/dev/null || echo 0)
|
|
local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md"
|
|
if [ -n "$_cache" ] && [ -f "$_cache" ]; then
|
|
local _c_line _c_status _c_ts _c_sig
|
|
_c_line=$(head -1 "$_cache" 2>/dev/null)
|
|
_c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1)
|
|
_c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2)
|
|
_c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3)
|
|
case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac
|
|
if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
|
|
echo "MODEL_OK (cached)"
|
|
return 0
|
|
fi
|
|
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry"
|
|
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
|
echo "MODEL_QUOTA_EXHAUSTED (cached)"
|
|
sed -n '2,4p' "$_cache" 2>/dev/null
|
|
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
|
|
return 4
|
|
fi
|
|
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
|
echo "MODEL_UNUSABLE (cached)"
|
|
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
|
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
|
return 1
|
|
fi
|
|
fi
|
|
local _out _code
|
|
_out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" </dev/null 2>&1)
|
|
_code=$?
|
|
if [ "$_code" -eq 0 ]; then
|
|
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
|
|
echo "MODEL_OK"
|
|
return 0
|
|
fi
|
|
# A usage limit is the account, not the model or the network: Codex says so
|
|
# plainly ("You've hit your usage limit ... try again at <time>"), and the
|
|
# inconclusive branch used to drop that line and re-probe in every skill.
|
|
# Only a failed, non-timeout call counts, and only Codex's own trailing error
|
|
# lines are read. Quota wins over rate limit: a 429 body that names
|
|
# insufficient_quota is the quota.
|
|
local _err _q_lines
|
|
if [ "$_code" -ne 124 ]; then
|
|
_err=$(printf '%s\n' "$_out" | awk '
|
|
/^[[:space:]]*(\[[^]]*\][[:space:]]*)?(ERROR:|stream error)/ { buf = buf $0 "\n"; next }
|
|
/^[[:space:]]*$/ { next }
|
|
{ buf = "" }
|
|
END { printf "%s", buf }')
|
|
fi
|
|
_q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3)
|
|
if [ -n "$_q_lines" ]; then
|
|
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_QUOTA_EXHAUSTED %s %s\n%s\n' "$_now" "$_sig" "$_q_lines" > "$_cache" 2>/dev/null; }
|
|
echo "MODEL_QUOTA_EXHAUSTED"
|
|
printf '%s\n' "$_q_lines"
|
|
echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted"
|
|
_gstack_codex_log_event "codex_quota_exhausted" 2>/dev/null || true
|
|
return 4
|
|
fi
|
|
_q_lines=$(printf '%s' "$_err" | grep -iE "rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)" | head -3)
|
|
if [ -n "$_q_lines" ]; then
|
|
_GSTACK_CODEX_PROBE_STATE="rate_limited"
|
|
echo "MODEL_PROBE_RATE_LIMITED — CODEX_MODE: unverified (rate_limited); proceeding, and the review's own result still decides."
|
|
printf '%s\n' "$_q_lines"
|
|
echo "HINT: Codex rate-limited the check (Codex's line above). Not cached; if the review itself is rate-limited, it reports missing coverage. $_docs#codex-rate-limited"
|
|
_gstack_codex_log_event "codex_rate_limited" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
# A model rejection is deterministic: a 400 ("is not supported", "requires a
|
|
# newer version of Codex") or a 404. A retired model answers 404 "does not
|
|
# exist or you do not have access to it", and a custom provider with a wrong
|
|
# base_url answers a bare 404; neither is network luck, so neither may fall
|
|
# through to the inconclusive branch below and report ready (#2843).
|
|
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
|
|
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
|
|
echo "MODEL_UNUSABLE"
|
|
printf '%s\n' "$_out" | grep -iE "model|404" | head -3
|
|
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
|
if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then
|
|
echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)."
|
|
_gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
|
|
elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
|
|
echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
|
|
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
|
|
_gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
|
|
else
|
|
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
|
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
|
|
fi
|
|
return 1
|
|
fi
|
|
# A CLI that cannot execute is deterministic, not transient: the fail-open
|
|
# below exists for network luck, and swallowing this here is what let a
|
|
# missing vendor binary report CODEX_MODE: ready while every Codex pass was
|
|
# silently skipped (#2742). 126 = found but not executable, 127 = not found.
|
|
# String signatures only count on a FAILED, NON-TIMEOUT spawn: a successful
|
|
# response that mentions "permission denied" must not classify as broken,
|
|
# and neither may a timed-out (124) probe whose partial output quotes such
|
|
# strings — 124 keeps its fail-open contract below.
|
|
_BROKEN_SIG='ENOENT|ENOEXEC|EACCES|no such file or directory|cannot execute binary file|not executable|permission denied'
|
|
if [ "$_code" -eq 126 ] || [ "$_code" -eq 127 ] || { [ "$_code" -ne 0 ] && [ "$_code" -ne 124 ] && printf '%s' "$_out" | grep -qiE "$_BROKEN_SIG"; }; then
|
|
echo "MODEL_UNUSABLE_INSTALL"
|
|
printf '%s\n' "$_out" | grep -iE "$_BROKEN_SIG" | head -3
|
|
echo "HINT: the Codex CLI is on PATH but cannot run — its binary or vendor payload is missing."
|
|
echo "HINT: reinstall with: npm install -g @openai/codex"
|
|
_gstack_codex_log_event "codex_broken_install" 2>/dev/null || true
|
|
return 2
|
|
fi
|
|
# Timeout (124) or transient failure: fail-open with a warning. The probe
|
|
# exists to catch the deterministic model 400, not to gate on network luck.
|
|
# Callers report CODEX_MODE: unverified, never ready; the exit stays 0 so
|
|
# skills rendered before this marker keep running the review.
|
|
_GSTACK_CODEX_PROBE_STATE="inconclusive"
|
|
echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — CODEX_MODE: unverified; proceeding, and the review's own result is still checked. If invocations fail with a model 400, see the codex skill's Error Handling entry."
|
|
return 0
|
|
}
|
|
|
|
# --- Version check ----------------------------------------------------------
|
|
|
|
_gstack_codex_version_check() {
|
|
# Warn on known-bad Codex CLI versions. Anchored regex prevents false
|
|
# positives like 0.120.10 or 0.120.20 from matching. 0.120.2-beta still
|
|
# matches the bad release and gets warned (it IS buggy).
|
|
# Update this list when a new Codex CLI version regresses.
|
|
local _ver _vcode
|
|
# Capture the code from codex, not from `head` — a pipeline reports the LAST
|
|
# command's status, which is why a CLI that only ever printed a spawn error
|
|
# still read as healthy here (#2742). Keep stderr: it carries the diagnosis.
|
|
_ver=$(codex --version 2>&1)
|
|
_vcode=$?
|
|
_ver=$(printf '%s' "$_ver" | head -1)
|
|
# Only a NON-ZERO exit is evidence of a broken CLI. Empty-but-successful
|
|
# output stays silent by design (a CLI may legitimately print nothing), which
|
|
# the "empty output → OK" case in this file's suite pins.
|
|
if [ "$_vcode" -ne 0 ]; then
|
|
echo "WARN: \`codex --version\` failed (exit $_vcode) — the CLI is on PATH but may not be runnable."
|
|
[ -n "$_ver" ] && echo "WARN: it said: $_ver"
|
|
echo "WARN: if Codex passes are being skipped, reinstall with: npm install -g @openai/codex"
|
|
_gstack_codex_log_event "codex_version_unreadable" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
[ -z "$_ver" ] && return 0
|
|
if echo "$_ver" | grep -Eq '(^|[^0-9.])0\.120\.(0|1|2)([^0-9.]|$)'; then
|
|
echo "WARN: Codex CLI $_ver has known stdin deadlock bugs. Run: npm install -g @openai/codex@latest"
|
|
_gstack_codex_log_event "codex_version_warning"
|
|
fi
|
|
}
|
|
|
|
# --- Timeout wrapper --------------------------------------------------------
|
|
|
|
_gstack_codex_timeout_wrapper() {
|
|
# Resolve wrapper binary: prefer gtimeout (Homebrew coreutils on macOS),
|
|
# fall back to timeout (Linux), else a bash-native watchdog. Arguments:
|
|
# $1 is the duration in seconds; rest is the command to run.
|
|
# A child that ignores TERM is KILLed after a grace period
|
|
# (_GSTACK_CODEX_KILL_AFTER seconds, default 10), so the deadline always
|
|
# ends the provider before the caller's outer tool gate (#2776). Either
|
|
# way a deadline reports timeout(1)'s exit 124; output already written stays.
|
|
local _duration="$1"
|
|
shift
|
|
local _grace="${_GSTACK_CODEX_KILL_AFTER:-10}"
|
|
local _to _rc _start
|
|
_to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "")
|
|
if [ -n "$_to" ]; then
|
|
_start=$(date +%s 2>/dev/null || echo 0)
|
|
"$_to" -k "$_grace" "$_duration" "$@"
|
|
_rc=$?
|
|
# 137 = timeout(1)'s KILL escalation; it is still the deadline firing.
|
|
if [ "$_rc" -eq 137 ] && [ $(( $(date +%s 2>/dev/null || echo 0) - _start )) -ge "${_duration%%.*}" ]; then
|
|
_rc=124
|
|
fi
|
|
return "$_rc"
|
|
else
|
|
# Stock macOS ships neither coreutils gtimeout nor timeout(1); running
|
|
# unwrapped let a hung `codex exec` block the probe — and the calling
|
|
# workflow — indefinitely. Emulate: background the command, TERM it at
|
|
# the deadline, KILL it after the grace period if it is still alive, and
|
|
# mirror timeout(1)'s exit-124 contract. The watchdog's stdout is
|
|
# detached so an early finish never blocks a caller's $(...) capture on
|
|
# the orphaned sleep.
|
|
# Without job control a background command's stdin is /dev/null; keep the
|
|
# caller's stdin so prompts fed with `codex exec -` arrive (#1674).
|
|
"$@" <&0 &
|
|
local _cmd_pid=$!
|
|
(
|
|
sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0 # best-effort: the command already exited, nothing to stop
|
|
_GSTACK_CODEX_WAITED=0
|
|
while [ "$_GSTACK_CODEX_WAITED" -lt "$_grace" ] && kill -0 "$_cmd_pid" 2>/dev/null; do
|
|
sleep 1
|
|
_GSTACK_CODEX_WAITED=$((_GSTACK_CODEX_WAITED + 1))
|
|
done
|
|
# Freeze the command first: killing its children while it runs lets it
|
|
# continue past the killed child and print output after its deadline.
|
|
kill -STOP "$_cmd_pid" 2>/dev/null
|
|
pkill -KILL -P "$_cmd_pid" 2>/dev/null
|
|
kill -KILL "$_cmd_pid" 2>/dev/null
|
|
exit 124
|
|
) >/dev/null 2>&1 &
|
|
local _watch_pid=$!
|
|
wait "$_cmd_pid"
|
|
_rc=$?
|
|
kill "$_watch_pid" 2>/dev/null
|
|
wait "$_watch_pid" 2>/dev/null
|
|
if [ "$?" -eq 124 ]; then
|
|
_rc=124
|
|
fi
|
|
return "$_rc"
|
|
fi
|
|
}
|
|
|
|
# --- Telemetry event --------------------------------------------------------
|
|
|
|
_gstack_codex_log_event() {
|
|
# Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl.
|
|
# Gated on $_TEL != "off" (caller sets this from gstack-config).
|
|
# Event types: codex_timeout, codex_auth_failed, codex_cli_missing,
|
|
# codex_version_warning, codex_model_unusable, codex_sandbox_unavailable,
|
|
# codex_model_retired, codex_cli_outdated, codex_quota_exhausted,
|
|
# codex_rate_limited.
|
|
# Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt
|
|
# content, env var values, or auth tokens.
|
|
local _event="$1"
|
|
local _duration="${2:-0}"
|
|
[ "${_TEL:-off}" = "off" ] && return 0
|
|
local _root
|
|
_root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
|
|
[ -n "$_root" ] || return 0
|
|
mkdir -p "$_root/analytics" 2>/dev/null || return 0
|
|
local _ts
|
|
_ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)
|
|
printf '{"skill":"codex","event":"%s","duration_s":"%s","ts":"%s"}\n' \
|
|
"$_event" "$_duration" "$_ts" \
|
|
>> "$_root/analytics/skill-usage.jsonl" 2>/dev/null || true
|
|
}
|
|
|
|
# --- Learnings log on hang --------------------------------------------------
|
|
|
|
_gstack_codex_log_hang() {
|
|
# Invoked when a codex invocation times out (exit 124). Records an
|
|
# operational learning so future /investigate sessions surface the pattern.
|
|
# Best-effort: errors swallowed.
|
|
local _mode="${1:-unknown}"
|
|
local _prompt_size="${2:-0}"
|
|
local _log_bin="$HOME/.claude/skills/gstack/bin/gstack-learnings-log"
|
|
[ -x "$_log_bin" ] || return 0
|
|
local _key="codex-hang-$(date +%s 2>/dev/null || echo unknown)"
|
|
"$_log_bin" "$(printf '{"skill":"codex","type":"operational","key":"%s","insight":"Codex hit its timeout wrapper during [%s] invocation. Prompt size: %s. Consider splitting prompt or checking network.","confidence":8,"source":"observed","files":["codex/SKILL.md.tmpl","autoplan/SKILL.md.tmpl"]}' "$_key" "$_mode" "$_prompt_size")" \
|
|
>/dev/null 2>&1 || true
|
|
}
|