mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
Garry Tan
92cfd07a79
* feat: add configurable model tiers and advisory freshness * fix: disclose role selections before Claude review execution * fix: reject malformed model policies and keep model pins portable * fix: separate readiness diagnostics and bound version checks * fix: preserve actionable model failures and validate review inputs before probes * fix: retry released probe locks and retain late-rejection repair context * feat: finalize configurable model tiers and verified release repairs * fix: preserve strict model policy reads on Windows * fix: keep fixture probes read-only and clarify pending CEO decisions * fix: verify native terminal closure before bounded descriptor settlement
918 lines
49 KiB
Bash
Executable File
918 lines
49 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# gstack-codex-probe: shared helper for /codex, /autoplan and every outside
|
|
# review. Run it as a command, one subcommand per probe (`gstack-codex-probe
|
|
# help`): it prints KEY: value lines on stdout, human status lines where they
|
|
# went before, and sets nothing in the caller's shell.
|
|
#
|
|
# Subcommands (each wraps one function below):
|
|
# select-model <exec|review> [--model <id>] CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:
|
|
# check-auth exit 0 authed, 1 not
|
|
# show-sandbox CODEX_SANDBOX:
|
|
# check-sandbox exit 0, or 3 with the reason on stderr
|
|
# probe-model <exec|review> [--model <id>] CODEX_PROBE_STATE:; exit 0, 1, 2 or 4
|
|
# role-ready <exec|review> [--model <id>] [--cwd <dir>]
|
|
# plan-review select, auth, sandbox and probe once:
|
|
# CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:,
|
|
# CODEX_PROBE_STATE:; exit 0, 1, 2, 3 or 4
|
|
# check-version warnings on stderr; exit 0
|
|
# show-first-use-notice notice on stderr; exit 0
|
|
# run-with-timeout <secs> <cmd...> the command's exit code; 124 on timeout
|
|
# log-event <event> [detail] silent; exit 0
|
|
# log-hang <skill> <prompt-size> silent; exit 0
|
|
# help usage; an unknown subcommand exits 64
|
|
#
|
|
# One deadline: an exported _CODEX_DEADLINE (epoch seconds) caps every
|
|
# supervised command at the time left, and one that would start after it does
|
|
# not run (exit 124). role-ready spends at most half of that time.
|
|
#
|
|
# Sourcing (compatibility window): skills rendered before the executed form
|
|
# source this file and call the functions directly. That still works, prints
|
|
# one deprecation line on stderr, and is removed in a release on or after
|
|
# 2026-10-21 together with the zsh self-locate block below.
|
|
#
|
|
# Functions (all prefixed with _gstack_codex_ for namespace hygiene):
|
|
# _gstack_codex_auth_probe — multi-signal auth check (env + file)
|
|
# _gstack_codex_select_model — resolve + print the runtime model (#2914)
|
|
# _gstack_codex_sandbox_mode — read-only, or full access only for GSTACK_CODEX_NO_SANDBOX=1
|
|
# _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux)
|
|
# _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
|
|
# _gstack_codex_model_probe — round-trip probe of the selected model (#2477);
|
|
# exit 4 = the account's Codex usage limit
|
|
# _gstack_codex_lock_acquire — bounded mkdir lock with abandoned-owner recovery
|
|
# _gstack_codex_role_ready — plan-review selection, auth, sandbox and model probe (role-ready)
|
|
# _gstack_codex_version_check — warn on known-bad Codex CLI versions
|
|
# _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
|
|
# _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/
|
|
#
|
|
# Hygiene rules for the sourced form (enforced by test/codex-hardening.test.ts):
|
|
# - Never change -e / -u / traps / IFS / PATH in the caller shell.
|
|
# - All internal vars prefix with _GSTACK_CODEX_.
|
|
# - All functions prefix with _gstack_codex_.
|
|
# - No command execution at source time (only function defs, the shared
|
|
# self-location block below, which is parameter expansion only, and the
|
|
# deprecation line; its _gstack_helper* variables are the one exception to
|
|
# the prefix rule and are unset once _GSTACK_CODEX_BIN is set).
|
|
|
|
# --- Self-location (bash and zsh) -------------------------------------------
|
|
# Under zsh BASH_SOURCE is empty, so every "$dir/../scripts/..." path became
|
|
# "/..." and Codex model selection failed. The shared block below sets
|
|
# _gstack_helper_dir; inside a function zsh's %x names the function, not the
|
|
# file, so it is copied once here.
|
|
_gstack_helper=gstack-codex-probe
|
|
# === gstack self-locate (shared, byte-identical in every sourced helper) ===
|
|
# Skill blocks source helpers into the host's shell: bash, or zsh on macOS
|
|
# (Claude Code's Bash tool included). Resolved once at source time, using
|
|
# parameter expansion only: BASH_SOURCE under bash, %x under zsh (inside eval
|
|
# so bash never parses it), made absolute with $PWD; else $GSTACK_ROOT/bin
|
|
# when it holds this helper. Any other case, and any other shell, fails loudly
|
|
# instead of guessing; the message and the return come before any bash- or
|
|
# zsh-only syntax, so dash and sh print it instead of a syntax error.
|
|
_gstack_helper_dir=""
|
|
_gstack_helper_shell=""
|
|
if [ -n "${BASH_VERSION:-}" ]; then
|
|
_gstack_helper_shell=bash
|
|
_gstack_helper_dir="${BASH_SOURCE[0]}"
|
|
elif [ -n "${ZSH_VERSION:-}" ]; then
|
|
_gstack_helper_shell=zsh
|
|
eval '_gstack_helper_dir="${(%):-%x}"'
|
|
fi
|
|
case "$_gstack_helper_dir" in
|
|
/*) _gstack_helper_dir="${_gstack_helper_dir%/*}" ;;
|
|
*/*) _gstack_helper_dir="$PWD/${_gstack_helper_dir%/*}" ;;
|
|
?*) _gstack_helper_dir="$PWD" ;;
|
|
esac
|
|
if [ -n "$_gstack_helper_shell" ] && [ ! -f "$_gstack_helper_dir/$_gstack_helper" ]; then
|
|
_gstack_helper_dir=""
|
|
if [ -n "${GSTACK_ROOT:-}" ] && [ -f "$GSTACK_ROOT/bin/$_gstack_helper" ]; then
|
|
_gstack_helper_dir="$GSTACK_ROOT/bin"
|
|
fi
|
|
fi
|
|
if [ -z "$_gstack_helper_shell" ] || [ -z "$_gstack_helper_dir" ]; then
|
|
_gstack_helper_error="gstack: cannot locate $_gstack_helper (shell: ${_gstack_helper_shell:-${0##*/}}). Source it from bash or zsh, or export GSTACK_ROOT=<install dir>. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location"
|
|
printf '%s\n' "$_gstack_helper_error" >&2
|
|
return 1 2>/dev/null || exit 1
|
|
fi
|
|
# === end gstack self-locate ===
|
|
_GSTACK_CODEX_BIN="$_gstack_helper_dir"
|
|
unset _gstack_helper _gstack_helper_dir _gstack_helper_shell
|
|
|
|
# --- Auth probe -------------------------------------------------------------
|
|
|
|
_gstack_codex_auth_probe() {
|
|
# Multi-signal: env vars OR auth file OR a custom provider's env_key. Avoids
|
|
# false negatives for env-auth users (CI, platform engineers, self-hosted
|
|
# OpenAI-compatible providers) that a file-only check would reject.
|
|
local _codex_home="${CODEX_HOME:-$HOME/.codex}"
|
|
# Use `-n` which returns true only for non-empty non-whitespace. Bash's [ -n ]
|
|
# alone allows whitespace; pair with a whitespace strip for robustness.
|
|
local _k1 _k2
|
|
_k1=$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')
|
|
_k2=$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')
|
|
if [ -n "$_k1" ] || [ -n "$_k2" ] || [ -f "$_codex_home/auth.json" ]; then
|
|
echo "AUTH_OK"
|
|
return 0
|
|
fi
|
|
# Custom OpenAI-compatible providers (#2192): a [model_providers.<id>]
|
|
# table names its credential variable in env_key. Only keys inside those
|
|
# tables count, comments are ignored, and the name must be a plain shell
|
|
# identifier before it is dereferenced.
|
|
local _env_key _resolved _missing=""
|
|
if [ -f "$_codex_home/config.toml" ]; then
|
|
while IFS= read -r _env_key; do
|
|
case "$_env_key" in ''|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac
|
|
# Indirect read through eval, not ${!name}: zsh rejects bash's
|
|
# indirection, and _env_key is a validated identifier by this point.
|
|
eval "_resolved=\${$_env_key:-}"
|
|
_resolved=$(printf '%s' "$_resolved" | tr -d '[:space:]')
|
|
if [ -n "$_resolved" ]; then
|
|
echo "AUTH_OK"
|
|
return 0
|
|
fi
|
|
_missing="${_missing:+$_missing, }$_env_key"
|
|
done < <(awk '/^[[:space:]]*#/ { next }
|
|
/^[[:space:]]*\[/ { in_provider = ($0 ~ /^[[:space:]]*\[model_providers\.[^]]+\][[:space:]]*(#.*)?$/); next }
|
|
in_provider && /^[[:space:]]*env_key[[:space:]]*=/ { v = $0; sub(/^[^=]*=[[:space:]]*/, "", v); sub(/[[:space:]]*#.*$/, "", v); gsub(/["\047[:space:]]/, "", v); print v }' "$_codex_home/config.toml" 2>/dev/null)
|
|
fi
|
|
echo "AUTH_FAILED"
|
|
[ -n "$_missing" ] && echo "HINT: $_codex_home/config.toml names a custom provider key ($_missing), but it is not set in this shell. Export it, or run codex login." >&2
|
|
return 1
|
|
}
|
|
|
|
# --- Runtime model selection (#2914) ---------------------------------------
|
|
|
|
_gstack_codex_select_model() {
|
|
# One selection record per invocation kind, shared by the probe and the
|
|
# dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL,
|
|
# Codex config.toml (review_model first for native review), gstack default.
|
|
# With $3 = plan-review the plan-review policy ranks the choice instead and
|
|
# its record carries role, tier, source and source-aware repair; $4 names the
|
|
# working-directory context forwarded to the shared role resolver.
|
|
# The TOML is read by the TypeScript resolver; bash never parses it.
|
|
local _kind="${1:-exec}" _explicit="${2:-}" _role="${3:-}" _cwd="${4:-}" _line _rc _rest
|
|
case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac
|
|
case "$_role" in ''|plan-review) ;; *) echo "ERROR: _gstack_codex_select_model role must be plan-review" >&2; return 2 ;; esac
|
|
if ! command -v bun >/dev/null 2>&1; then
|
|
echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2
|
|
return 1
|
|
fi
|
|
# Function-local argv: each value stays one literal argument in bash and zsh.
|
|
set -- --runtime "$_kind"
|
|
[ -n "$_role" ] && set -- "$@" --role "$_role"
|
|
[ -n "$_explicit" ] && set -- "$@" --explicit "$_explicit"
|
|
[ -n "$_cwd" ] && set -- "$@" --cwd "$_cwd"
|
|
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" "$@")
|
|
_rc=$?
|
|
_GSTACK_CODEX_SEL="${_line%%$'\t'*}"
|
|
_rest="${_line#*$'\t'}"
|
|
_GSTACK_CODEX_SEL_TIER=""
|
|
_GSTACK_CODEX_SEL_REPAIR=""
|
|
if [ -n "$_role" ]; then
|
|
_GSTACK_CODEX_SEL_SRC="${_rest%%$'\t'*}"
|
|
_rest="${_rest#*$'\t'}"
|
|
_GSTACK_CODEX_SEL_TIER="${_rest%%$'\t'*}"
|
|
_GSTACK_CODEX_SEL_REPAIR="${_rest#*$'\t'}"
|
|
else
|
|
_GSTACK_CODEX_SEL_SRC="$_rest"
|
|
fi
|
|
case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac
|
|
if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then
|
|
_GSTACK_CODEX_SEL=""
|
|
_GSTACK_CODEX_SEL_SRC=""
|
|
_GSTACK_CODEX_SEL_ROLE=""
|
|
echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2
|
|
return 1
|
|
fi
|
|
_GSTACK_CODEX_SEL_KIND="$_kind"
|
|
_GSTACK_CODEX_SEL_ROLE="$_role"
|
|
_GSTACK_CODEX_SEL_CWD="$_cwd"
|
|
if [ -n "$_role" ]; then
|
|
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; role: $_role, tier: $_GSTACK_CODEX_SEL_TIER; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
|
else
|
|
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
|
fi
|
|
_gstack_codex_sandbox_mode
|
|
}
|
|
|
|
_gstack_codex_role_ready() {
|
|
# $1 exec|review, $2 target repository, $3 a model named for this request.
|
|
# Resolves the plan-review model once and checks auth, the sandbox and that
|
|
# model; the caller dispatches the same record. Under _CODEX_DEADLINE these
|
|
# checks get the first half of the time left, so the dispatch keeps the rest.
|
|
local _end="${_CODEX_DEADLINE:-}" _now
|
|
case "$_end" in
|
|
''|*[!0-9]*) local _CODEX_DEADLINE="" ;;
|
|
*) _now=$(date +%s 2>/dev/null || echo 0); local _CODEX_DEADLINE=$(( _now + (_end - _now) / 2 )) ;;
|
|
esac
|
|
_gstack_codex_select_model "$1" "${3:-}" plan-review "${2:-}" || return 1
|
|
_gstack_codex_auth_probe || return 1
|
|
_gstack_codex_sandbox_preflight || return $?
|
|
_gstack_codex_version_check
|
|
_gstack_codex_model_probe "$1" || return $?
|
|
[ -z "${_GSTACK_CODEX_SEL_REPAIR:-}" ] || echo "HINT: If the review rejects the selected model, repair its winning source: $_GSTACK_CODEX_SEL_REPAIR."
|
|
return 0
|
|
}
|
|
|
|
# --- Sandbox (B1) -----------------------------------------------------------
|
|
|
|
_gstack_codex_sandbox_mode() {
|
|
# Every gstack Codex command passes -s / sandbox_mode from this variable, so
|
|
# one place decides it. Codex has no read-only mode without its sandbox, so
|
|
# the escape hatch is full access: honored only for exactly 1, only from the
|
|
# shell environment (never a dotenv file or config), and announced each use.
|
|
if [ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ]; then
|
|
_GSTACK_CODEX_SANDBOX="danger-full-access"
|
|
echo "WARNING: GSTACK_CODEX_NO_SANDBOX=1: Codex runs this review without a sandbox and can read and write anything your user can. Use it only inside a container you trust." >&2
|
|
else
|
|
_GSTACK_CODEX_SANDBOX="read-only"
|
|
fi
|
|
}
|
|
|
|
_gstack_codex_sandbox_preflight() {
|
|
# Free check before the first paid call: `codex sandbox` starts the same
|
|
# bubblewrap sandbox with no model. Only a recognized sandbox failure blocks
|
|
# (exit 3, printed as missing coverage); success, a timeout, or an older CLI
|
|
# without the subcommand defers to the post-run classifier. Linux only, and
|
|
# skipped when the user turned the sandbox off. Fixtures:
|
|
# test/fixtures/codex-sandbox/.
|
|
[ "${GSTACK_CODEX_NO_SANDBOX:-}" = "1" ] && return 0
|
|
[ "$(uname -s 2>/dev/null)" = "Linux" ] || return 0
|
|
local _out _code _line
|
|
_out=$(_gstack_codex_timeout_wrapper 10 codex sandbox -c 'sandbox_mode="read-only"' true </dev/null 2>&1)
|
|
_code=$?
|
|
{ [ "$_code" -eq 0 ] || [ "$_code" -eq 124 ]; } && return 0
|
|
_line=$(printf '%s\n' "$_out" | grep -m1 -iE '^[[:space:]]*bwrap: |bubblewrap is unavailable|landlock.{0,60}(fail|error|not supported|unsupported)|seccomp.{0,60}(fail|error)|user namespaces?.{0,80}(not (allowed|permitted|supported)|denied|disabled)' | sed 's/^[[:space:]]*//' | cut -c1-240)
|
|
[ -n "$_line" ] || return 0
|
|
echo "CODEX_SANDBOX: unavailable"
|
|
echo "Codex outside review unavailable: Codex's sandbox could not start here ($_line). No review ran; this is missing coverage, not a pass. Fix: enable unprivileged user namespaces for this container, or set GSTACK_CODEX_NO_SANDBOX=1." >&2
|
|
_gstack_codex_log_event "codex_sandbox_unavailable" 2>/dev/null || true
|
|
return 3
|
|
}
|
|
|
|
# --- First-use notice (#965) ------------------------------------------------
|
|
|
|
_gstack_codex_first_use_notice() {
|
|
# Outside reviews are on by default and use whatever Codex login exists, so
|
|
# the first one on a machine says where the code goes, with which account,
|
|
# and how to turn reviews off. Shown once per state root; never blocks and
|
|
# never prints a credential, only where it comes from.
|
|
local _root _marker _codex_home="${CODEX_HOME:-$HOME/.codex}" _account _provider
|
|
_root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
|
|
[ -n "$_root" ] || return 0
|
|
_marker="$_root/.codex-review-notice-shown"
|
|
[ -e "$_marker" ] && return 0
|
|
if [ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in CODEX_API_KEY"
|
|
elif [ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ]; then _account="the API key in OPENAI_API_KEY"
|
|
elif grep -q '"auth_mode"[[:space:]]*:[[:space:]]*"chatgpt"' "$_codex_home/auth.json" 2>/dev/null; then _account="the ChatGPT account Codex is logged in with (see: codex login status)"
|
|
elif [ -f "$_codex_home/auth.json" ]; then _account="the API key saved by codex login in $_codex_home/auth.json"
|
|
else _account="the credentials your Codex config.toml provider names"
|
|
fi
|
|
_provider=$(sed -n 's/^[[:space:]]*model_provider[[:space:]]*=[[:space:]]*"\([A-Za-z0-9_.-]*\)".*/\1/p' "$_codex_home/config.toml" 2>/dev/null | head -1)
|
|
echo "NOTICE: gstack outside reviews send the review prompt and code to Codex (provider: ${_provider:-openai}) using $_account. Reviews stay on; this notice shows once per machine. To turn them off: gstack-config set codex_reviews disabled" >&2
|
|
mkdir -p "$_root" 2>/dev/null && : > "$_marker" 2>/dev/null
|
|
return 0
|
|
}
|
|
|
|
# --- Model round-trip probe (#2477) ------------------------------------------
|
|
|
|
_gstack_codex_model_probe() {
|
|
# Auth-exists is a weaker signal than the auth probe implies: a ChatGPT
|
|
# account can be valid while the model gstack will request is unavailable.
|
|
# A short real round trip with the selected model catches model rejection
|
|
# and entitlement changes in one shot (#2477). The model comes from
|
|
# _gstack_codex_select_model ($1 = exec|review, default exec), the same
|
|
# record the dispatch flag uses, so the probe never checks a different model.
|
|
#
|
|
# Contract:
|
|
# MODEL_OK (exit 0) — round trip succeeded; cached 1h.
|
|
# MODEL_UNUSABLE (exit 1) — deterministic model rejection (400, or a
|
|
# 404 from a retired model or a wrong custom-provider base_url) or an
|
|
# invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every
|
|
# preflight charged the affected user a 30s round trip + real tokens per
|
|
# review section, forever. Changing the model or config.toml changes the
|
|
# cache signature and re-probes immediately; the short TTL covers
|
|
# server-side entitlement recovery the signature can't see.
|
|
# MODEL_UNUSABLE_INSTALL (exit 2) — the CLI cannot execute at all (spawn
|
|
# ENOENT, non-executable binary, missing vendor payload). Deterministic,
|
|
# so fail-open is wrong: retrying never helps. Never cached — a reinstall
|
|
# fixes it and must be picked up on the very next probe (#2742).
|
|
# MODEL_QUOTA_EXHAUSTED (exit 4) — Codex's usage-limit or insufficient_quota
|
|
# error on a failed, non-timeout call: the account, not the model. Codex's
|
|
# own line (with its reset time) is relayed verbatim; cached 15 min under
|
|
# the same signature as MODEL_UNUSABLE, so later fresh-shell blocks in the
|
|
# run make no Codex call. GSTACK_CODEX_PROBE_RETRY=1 skips both negative
|
|
# cache entries for that probe.
|
|
# MODEL_PROBE_RATE_LIMITED (exit 0) — a plain 429 / rate-limit error: often
|
|
# gone in seconds, so the review still runs, like the inconclusive case
|
|
# (_GSTACK_CODEX_PROBE_STATE=rate_limited, CODEX_MODE: unverified
|
|
# (rate_limited)). Codex's line is relayed verbatim; never cached.
|
|
# MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still
|
|
# runs so a slow network never wedges codex mode, but readiness is
|
|
# unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE:
|
|
# unverified) and the review's own classification decides. Never cached.
|
|
#
|
|
# Quota and rate-limit signatures read only Codex's own trailing error lines
|
|
# (ERROR: / stream error), never the whole output: it echoes the prompt.
|
|
#
|
|
# Only call this AFTER _gstack_codex_auth_probe passes — probing without
|
|
# auth just measures the auth failure again.
|
|
local _kind="${1:-exec}"
|
|
if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then
|
|
if ! _gstack_codex_select_model "$_kind" "" "${_GSTACK_CODEX_SEL_ROLE:-}" "${_GSTACK_CODEX_SEL_CWD:-}"; then
|
|
echo "MODEL_UNUSABLE"
|
|
echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
|
|
return 1
|
|
fi
|
|
fi
|
|
local _model="$_GSTACK_CODEX_SEL"
|
|
local _codex_home="${CODEX_HOME:-$HOME/.codex}"
|
|
# State root from the shared twin (sourced in a subshell, so the caller's
|
|
# shell is untouched). A missing twin only disables the cache.
|
|
local _gstack_home _cache=""
|
|
_gstack_home="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || _gstack_home=""
|
|
[ -n "$_gstack_home" ] && _cache="$_gstack_home/.codex-model-probe"
|
|
# Cache signature (#2787): everything that can change the round trip's
|
|
# outcome without the hour passing — the model, the Codex home and its
|
|
# config.toml mtime, the auth mode (which env var or auth.json, plus its
|
|
# mtime; never a credential value), and the resolved codex binary and its
|
|
# mtime (an upgrade or PATH swap re-probes). Folded through cksum so paths
|
|
# with spaces stay one field of the cache line.
|
|
# GNU-first stat order + numeric validation (the #2195 pattern): on GNU
|
|
# stat, `-f` means FILESYSTEM mode, so the BSD-first form emitted a
|
|
# multi-line filesystem block on Linux — the signature then never matched
|
|
# its own cache line and the cache missed on every read. BSD stat rejects
|
|
# `-c` cleanly, so GNU-first degrades correctly on macOS.
|
|
local _cfg_m _auth_m _auth_mode="" _bin _bin_m _sig
|
|
_cfg_m=$(stat -c %Y "$_codex_home/config.toml" 2>/dev/null || stat -f %m "$_codex_home/config.toml" 2>/dev/null || echo 0)
|
|
_auth_m=$(stat -c %Y "$_codex_home/auth.json" 2>/dev/null || stat -f %m "$_codex_home/auth.json" 2>/dev/null || echo 0)
|
|
case "$_cfg_m" in ''|*[!0-9]*) _cfg_m=0 ;; esac
|
|
case "$_auth_m" in ''|*[!0-9]*) _auth_m=0 ;; esac
|
|
[ -n "$(printf '%s' "${CODEX_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-codex,"
|
|
[ -n "$(printf '%s' "${OPENAI_API_KEY:-}" | tr -d '[:space:]')" ] && _auth_mode="${_auth_mode}env-openai,"
|
|
[ -f "$_codex_home/auth.json" ] && _auth_mode="${_auth_mode}file,"
|
|
_bin=$(command -v codex 2>/dev/null || echo none)
|
|
_bin=$(readlink -f "$_bin" 2>/dev/null || printf '%s' "$_bin")
|
|
_bin_m=$(stat -c %Y "$_bin" 2>/dev/null || stat -f %m "$_bin" 2>/dev/null || echo 0)
|
|
case "$_bin_m" in ''|*[!0-9]*) _bin_m=0 ;; esac
|
|
_sig=$(printf '%s\n' "$_model" "$_codex_home" "$_cfg_m" "$_auth_mode" "$_auth_m" "$_bin" "$_bin_m" | cksum | tr -d ' ')
|
|
local _now
|
|
_now=$(date +%s 2>/dev/null || echo 0)
|
|
local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md"
|
|
# Up to 16 signatures share one cache file. Same-signature misses serialize
|
|
# on a bounded lock and recheck the cache, so concurrent callers pay for one
|
|
# round trip. A lock timeout or unreadable cache is unverified readiness,
|
|
# never a duplicate paid probe; the review's own result still decides.
|
|
local _lock="" _rc _wait="${_GSTACK_CODEX_LOCK_WAIT:-45}"
|
|
# Under a deadline, wait only while the 30-second round trip still fits.
|
|
case "${_CODEX_DEADLINE:-}" in
|
|
''|*[!0-9]*) ;;
|
|
*) [ $((_CODEX_DEADLINE - _now - 30)) -lt "$_wait" ] && _wait=$((_CODEX_DEADLINE - _now - 30))
|
|
[ "$_wait" -ge 0 ] || _wait=0 ;;
|
|
esac
|
|
if [ -n "$_cache" ]; then
|
|
_gstack_codex_probe_cache_hit; _rc=$?
|
|
[ "$_rc" -lt 10 ] && return "$_rc"
|
|
if [ "$_rc" -eq 11 ]; then _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; fi
|
|
_lock="$_gstack_home/.codex-model-probe.locks/$_sig"
|
|
if ! mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null; then
|
|
_gstack_codex_probe_unverified "cannot create the probe lock under $_gstack_home"; return 0
|
|
fi
|
|
_gstack_codex_lock_acquire "$_lock" "$_wait" "${_GSTACK_CODEX_LOCK_STALE:-90}"; _rc=$?
|
|
if [ "$_rc" -eq 1 ]; then _gstack_codex_probe_unverified "another probe of this model still holds its lock"; return 0; fi
|
|
if [ "$_rc" -ne 0 ]; then _gstack_codex_probe_unverified "cannot record ownership of the probe lock under $_gstack_home"; return 0; fi
|
|
_now=$(date +%s 2>/dev/null || echo "$_now")
|
|
_gstack_codex_probe_cache_hit; _rc=$?
|
|
if [ "$_rc" -lt 10 ] || [ "$_rc" -eq 11 ]; then
|
|
_gstack_codex_lock_release "$_lock"
|
|
[ "$_rc" -eq 11 ] && { _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; }
|
|
return "$_rc"
|
|
fi
|
|
fi
|
|
_gstack_codex_probe_round_trip; _rc=$?
|
|
[ -n "$_lock" ] && _gstack_codex_lock_release "$_lock"
|
|
return "$_rc"
|
|
}
|
|
|
|
_gstack_codex_probe_round_trip() {
|
|
# The paid round trip and its classification. Called only by
|
|
# _gstack_codex_model_probe; reads its locals (_model, _sig, _cache, ...).
|
|
local _out _code
|
|
_out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" </dev/null 2>&1)
|
|
_code=$?
|
|
if [ "$_code" -eq 0 ]; then
|
|
_gstack_codex_probe_cache_store MODEL_OK
|
|
echo "MODEL_OK"
|
|
return 0
|
|
fi
|
|
# A usage limit is the account, not the model or the network: Codex says so
|
|
# plainly ("You've hit your usage limit ... try again at <time>"), and the
|
|
# inconclusive branch used to drop that line and re-probe in every skill.
|
|
# Only a failed, non-timeout call counts, and only Codex's own trailing error
|
|
# lines are read. Quota wins over rate limit: a 429 body that names
|
|
# insufficient_quota is the quota.
|
|
local _err _q_lines
|
|
if [ "$_code" -ne 124 ]; then
|
|
_err=$(printf '%s\n' "$_out" | awk '
|
|
/^[[:space:]]*(\[[^]]*\][[:space:]]*)?(ERROR:|stream error)/ { buf = buf $0 "\n"; next }
|
|
/^[[:space:]]*$/ { next }
|
|
{ buf = "" }
|
|
END { printf "%s", buf }')
|
|
fi
|
|
_q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3)
|
|
if [ -n "$_q_lines" ]; then
|
|
_gstack_codex_probe_cache_store MODEL_QUOTA_EXHAUSTED "$_q_lines"
|
|
echo "MODEL_QUOTA_EXHAUSTED"
|
|
printf '%s\n' "$_q_lines"
|
|
echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted"
|
|
_gstack_codex_log_event "codex_quota_exhausted" 2>/dev/null || true
|
|
return 4
|
|
fi
|
|
_q_lines=$(printf '%s' "$_err" | grep -iE "rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)" | head -3)
|
|
if [ -n "$_q_lines" ]; then
|
|
_GSTACK_CODEX_PROBE_STATE="rate_limited"
|
|
echo "MODEL_PROBE_RATE_LIMITED — CODEX_MODE: unverified (rate_limited); proceeding, and the review's own result still decides."
|
|
printf '%s\n' "$_q_lines"
|
|
echo "HINT: Codex rate-limited the check (Codex's line above). Not cached; if the review itself is rate-limited, it reports missing coverage. $_docs#codex-rate-limited"
|
|
_gstack_codex_log_event "codex_rate_limited" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
# A model rejection is deterministic: a 400 ("is not supported", "requires a
|
|
# newer version of Codex") or a 404. A retired model answers 404 "does not
|
|
# exist or you do not have access to it", and a custom provider with a wrong
|
|
# base_url answers a bare 404; neither is network luck, so neither may fall
|
|
# through to the inconclusive branch below and report ready (#2843).
|
|
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
|
|
_gstack_codex_probe_cache_store MODEL_UNUSABLE
|
|
echo "MODEL_UNUSABLE"
|
|
printf '%s\n' "$_out" | grep -iE "model|404" | head -3
|
|
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
|
if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then
|
|
echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)."
|
|
_gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
|
|
elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
|
|
echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
|
|
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
|
|
echo "HINT: choose a current model ($_GSTACK_CODEX_SEL_REPAIR), or fix that provider's base_url."
|
|
else
|
|
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
|
|
fi
|
|
_gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
|
|
else
|
|
_gstack_codex_model_repair_hint
|
|
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
|
|
fi
|
|
return 1
|
|
fi
|
|
# A CLI that cannot execute is deterministic, not transient: the fail-open
|
|
# below exists for network luck, and swallowing this here is what let a
|
|
# missing vendor binary report CODEX_MODE: ready while every Codex pass was
|
|
# silently skipped (#2742). 126 = found but not executable, 127 = not found.
|
|
# String signatures only count on a FAILED, NON-TIMEOUT spawn: a successful
|
|
# response that mentions "permission denied" must not classify as broken,
|
|
# and neither may a timed-out (124) probe whose partial output quotes such
|
|
# strings — 124 keeps its fail-open contract below.
|
|
_BROKEN_SIG='ENOENT|ENOEXEC|EACCES|no such file or directory|cannot execute binary file|not executable|permission denied'
|
|
if [ "$_code" -eq 126 ] || [ "$_code" -eq 127 ] || { [ "$_code" -ne 0 ] && [ "$_code" -ne 124 ] && printf '%s' "$_out" | grep -qiE "$_BROKEN_SIG"; }; then
|
|
echo "MODEL_UNUSABLE_INSTALL"
|
|
printf '%s\n' "$_out" | grep -iE "$_BROKEN_SIG" | head -3
|
|
echo "HINT: the Codex CLI is on PATH but cannot run — its binary or vendor payload is missing."
|
|
echo "HINT: reinstall with: npm install -g @openai/codex"
|
|
_gstack_codex_log_event "codex_broken_install" 2>/dev/null || true
|
|
return 2
|
|
fi
|
|
# Timeout (124) or transient failure: fail-open with a warning. The probe
|
|
# exists to catch the deterministic model 400, not to gate on network luck.
|
|
# Callers report CODEX_MODE: unverified, never ready; the exit stays 0 so
|
|
# skills rendered before this marker keep running the review.
|
|
_GSTACK_CODEX_PROBE_STATE="inconclusive"
|
|
echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — CODEX_MODE: unverified; proceeding, and the review's own result is still checked. If invocations fail with a model 400, see the codex skill's Error Handling entry."
|
|
return 0
|
|
}
|
|
|
|
_gstack_codex_model_repair_hint() {
|
|
# Repair for the record actually selected: a plan-review record names its
|
|
# winning source (request, env, tier override, catalog or host mode).
|
|
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
|
|
echo "HINT: choose a model your account can use: $_GSTACK_CODEX_SEL_REPAIR"
|
|
else
|
|
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in ${_codex_home:-\$CODEX_HOME}/config.toml, or name one for this request."
|
|
fi
|
|
}
|
|
|
|
_gstack_codex_probe_unverified() {
|
|
_GSTACK_CODEX_PROBE_STATE="inconclusive"
|
|
echo "MODEL_PROBE_INCONCLUSIVE ($1) — CODEX_MODE: unverified; no duplicate probe was made, and the review's own result is still checked."
|
|
}
|
|
|
|
_gstack_codex_probe_cache_hit() {
|
|
# Cache file: one "STATUS EPOCH SIGNATURE" line per entry, newest last, each
|
|
# followed by its tab-prefixed Codex lines. The pre-v2 one-entry file has the
|
|
# same first line, so it migrates by TTL; its unprefixed lines are ignored.
|
|
# Returns 0/1/4 after printing a hit, 10 on a miss, 11 when unreadable.
|
|
[ -e "$_cache" ] || return 10
|
|
{ [ -f "$_cache" ] && [ -r "$_cache" ]; } || return 11
|
|
local _hit _head _c_status _c_ts _detail=""
|
|
_hit=$(awk -v sig="$_sig" '
|
|
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ { cur = ($3 == sig); if (cur) out = $1 " " $2; next }
|
|
/^\t/ { if (cur) out = out "\n" substr($0, 2); next }
|
|
{ cur = 0 }
|
|
END { if (out != "") print out }' "$_cache" 2>/dev/null) || return 11
|
|
[ -n "$_hit" ] || return 10
|
|
_head="${_hit%%$'\n'*}"
|
|
[ "$_head" != "$_hit" ] && _detail="${_hit#*$'\n'}"
|
|
_c_status="${_head%% *}"
|
|
_c_ts="${_head#* }"
|
|
case "$_c_ts" in ''|*[!0-9]*) return 10 ;; esac
|
|
[ "$_c_ts" -le "$_now" ] || return 10
|
|
if [ "$_c_status" = "MODEL_OK" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
|
|
echo "MODEL_OK (cached)"
|
|
return 0
|
|
fi
|
|
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && return 10
|
|
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
|
echo "MODEL_QUOTA_EXHAUSTED (cached)"
|
|
[ -n "$_detail" ] && printf '%s\n' "$_detail"
|
|
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
|
|
return 4
|
|
fi
|
|
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
|
echo "MODEL_UNUSABLE (cached)"
|
|
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
|
_gstack_codex_model_repair_hint
|
|
return 1
|
|
fi
|
|
return 10
|
|
}
|
|
|
|
_gstack_codex_probe_cache_store() {
|
|
# $1 status, $2 optional Codex lines. Rewrites the file under a short write
|
|
# lock and publishes it with one rename: this signature's entry replaces any
|
|
# older one, expired entries drop, and at most 16 entries remain. A failure
|
|
# leaves the previous file intact; the probe's own result still stands.
|
|
# Timestamps are publication time, read under the write lock: a slow round
|
|
# trip must not judge entries other probes published meanwhile as future.
|
|
[ -n "$_cache" ] || return 0
|
|
[ -d "$_cache" ] && return 1
|
|
local _w="$_gstack_home/.codex-model-probe.locks/write" _tmp="$_cache.tmp.$$" _wrc=0 _pub
|
|
mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null || return 1
|
|
_gstack_codex_lock_acquire "$_w" 5 30 || return 1
|
|
_pub=$(date +%s 2>/dev/null || echo "$_now")
|
|
: > "$_tmp" 2>/dev/null || _wrc=1
|
|
if [ "$_wrc" -eq 0 ] && [ -f "$_cache" ]; then
|
|
awk -v now="$_pub" -v sig="$_sig" -v keep=15 '
|
|
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ {
|
|
ttl = ($1 == "MODEL_OK") ? 3600 : 900
|
|
cur = ($3 != sig && $2 <= now && now - $2 < ttl)
|
|
if (cur) entry[++n] = $0
|
|
next
|
|
}
|
|
/^\t/ { if (cur) entry[n] = entry[n] "\n" $0; next }
|
|
{ cur = 0 }
|
|
END { for (i = (n > keep ? n - keep + 1 : 1); i <= n; i++) print entry[i] }' "$_cache" > "$_tmp" 2>/dev/null || _wrc=1
|
|
fi
|
|
if [ "$_wrc" -eq 0 ]; then
|
|
{
|
|
printf '%s %s %s\n' "$1" "$_pub" "$_sig"
|
|
[ -z "${2:-}" ] || printf '%s\n' "$2" | awk '{ print "\t" $0 }'
|
|
} >> "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_cache" 2>/dev/null || _wrc=1
|
|
fi
|
|
rm -f "$_tmp" 2>/dev/null
|
|
_gstack_codex_lock_release "$_w"
|
|
return "$_wrc"
|
|
}
|
|
|
|
# --- Bounded lock -----------------------------------------------------------
|
|
|
|
_gstack_codex_lock_acquire() {
|
|
# $1 lock directory, $2 seconds to wait, $3 seconds after which even a live
|
|
# owner is abandoned. mkdir is atomic on every platform bash runs on. A dead
|
|
# owner on this host is recovered at once; a missing owner file is the
|
|
# creation window and becomes stale after 5 seconds. 0 = held, 1 = timed
|
|
# out, 2 = the lock or its owner record cannot be created.
|
|
local _dir="$1" _tries=$(( ${2:-45} * 10 )) _stale="${3:-90}" _host _owner _pid _ohost _ts _now _i=0 _dead
|
|
_host=$(uname -n 2>/dev/null || echo unknown)
|
|
while :; do
|
|
if mkdir "$_dir" 2>/dev/null; then
|
|
printf '%s %s %s\n' "$$" "$_host" "$(date +%s 2>/dev/null || echo 0)" > "$_dir/owner" 2>/dev/null && return 0
|
|
rm -f "$_dir/owner" 2>/dev/null
|
|
rmdir "$_dir" 2>/dev/null
|
|
return 2
|
|
fi
|
|
if [ ! -d "$_dir" ]; then
|
|
{ [ -e "$_dir" ] || [ -L "$_dir" ]; } && return 2
|
|
{ [ -d "${_dir%/*}" ] && [ -w "${_dir%/*}" ] && [ -x "${_dir%/*}" ]; } || return 2
|
|
[ "$_i" -ge "$_tries" ] && return 2
|
|
_i=$((_i + 1))
|
|
sleep 0.1
|
|
continue
|
|
fi
|
|
_now=$(date +%s 2>/dev/null || echo 0)
|
|
_owner=$(cat "$_dir/owner" 2>/dev/null)
|
|
_dead=0
|
|
if [ -n "$_owner" ]; then
|
|
read -r _pid _ohost _ts <<< "$_owner"
|
|
case "$_pid" in ''|*[!0-9]*) _pid=0 ;; esac
|
|
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
|
|
if [ "$_ohost" = "$_host" ] && ! kill -0 "$_pid" 2>/dev/null; then _dead=1; fi
|
|
[ $((_now - _ts)) -ge "$_stale" ] && _dead=1
|
|
else
|
|
_ts=$(stat -c %Y "$_dir" 2>/dev/null || stat -f %m "$_dir" 2>/dev/null || echo 0)
|
|
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
|
|
[ $((_now - _ts)) -ge 5 ] && _dead=1
|
|
fi
|
|
[ "$_dead" -eq 1 ] && _gstack_codex_lock_break "$_dir" "$_owner"
|
|
[ "$_i" -ge "$_tries" ] && return 1
|
|
_i=$((_i + 1))
|
|
sleep 0.1
|
|
done
|
|
}
|
|
|
|
_gstack_codex_lock_break() {
|
|
# Breakers serialize on $1.break and remove the lock only if it still has the
|
|
# owner they judged abandoned, so a lock re-taken meanwhile survives.
|
|
local _dir="$1" _seen="$2" _bt _now
|
|
if mkdir "$_dir.break" 2>/dev/null; then
|
|
[ "$(cat "$_dir/owner" 2>/dev/null)" = "$_seen" ] && rm -rf "$_dir" 2>/dev/null
|
|
rmdir "$_dir.break" 2>/dev/null
|
|
return 0
|
|
fi
|
|
_now=$(date +%s 2>/dev/null || echo 0)
|
|
_bt=$(stat -c %Y "$_dir.break" 2>/dev/null || stat -f %m "$_dir.break" 2>/dev/null || echo "$_now")
|
|
case "$_bt" in ''|*[!0-9]*) _bt="$_now" ;; esac
|
|
[ $((_now - _bt)) -ge 10 ] && rmdir "$_dir.break" 2>/dev/null
|
|
return 0
|
|
}
|
|
|
|
_gstack_codex_lock_release() {
|
|
# Remove only a lock this shell still owns.
|
|
local _host
|
|
_host=$(uname -n 2>/dev/null || echo unknown)
|
|
case "$(cat "$1/owner" 2>/dev/null)" in
|
|
"$$ $_host "*) rm -rf "$1" 2>/dev/null ;;
|
|
esac
|
|
return 0
|
|
}
|
|
|
|
# --- Version check ----------------------------------------------------------
|
|
|
|
_gstack_codex_version_check() {
|
|
# Warn on known-bad Codex CLI versions. Anchored regex prevents false
|
|
# positives like 0.120.10 or 0.120.20 from matching. 0.120.2-beta still
|
|
# matches the bad release and gets warned (it IS buggy).
|
|
# Update this list when a new Codex CLI version regresses.
|
|
local _ver _vcode
|
|
# Capture the code from codex, not from `head` — a pipeline reports the LAST
|
|
# command's status, which is why a CLI that only ever printed a spawn error
|
|
# still read as healthy here (#2742). Keep stderr: it carries the diagnosis.
|
|
_ver=$(_gstack_codex_timeout_wrapper 5 codex --version </dev/null 2>&1)
|
|
_vcode=$?
|
|
_ver=$(printf '%s' "$_ver" | head -1)
|
|
if [ "$_vcode" -eq 124 ]; then
|
|
echo 'WARN: `codex --version` timed out; version unknown. The invocation keeps its existing deadline.'
|
|
return 0
|
|
fi
|
|
# Only a NON-ZERO exit is evidence of a broken CLI. Empty-but-successful
|
|
# output stays silent by design (a CLI may legitimately print nothing), which
|
|
# the "empty output → OK" case in this file's suite pins.
|
|
if [ "$_vcode" -ne 0 ]; then
|
|
echo "WARN: \`codex --version\` failed (exit $_vcode) — the CLI is on PATH but may not be runnable."
|
|
[ -n "$_ver" ] && echo "WARN: it said: $_ver"
|
|
echo "WARN: if Codex passes are being skipped, reinstall with: npm install -g @openai/codex"
|
|
_gstack_codex_log_event "codex_version_unreadable" 2>/dev/null || true
|
|
return 0
|
|
fi
|
|
[ -z "$_ver" ] && return 0
|
|
if echo "$_ver" | grep -Eq '(^|[^0-9.])0\.120\.(0|1|2)([^0-9.]|$)'; then
|
|
echo "WARN: Codex CLI $_ver has known stdin deadlock bugs. Run: npm install -g @openai/codex@latest"
|
|
_gstack_codex_log_event "codex_version_warning"
|
|
fi
|
|
}
|
|
|
|
# --- Timeout wrapper --------------------------------------------------------
|
|
|
|
_gstack_codex_timeout_wrapper() {
|
|
# Resolve wrapper binary: prefer gtimeout (Homebrew coreutils on macOS),
|
|
# fall back to timeout (Linux), else a bash-native watchdog. Arguments:
|
|
# $1 is the duration in seconds; rest is the command to run.
|
|
# A child that ignores TERM is KILLed after a grace period
|
|
# (_GSTACK_CODEX_KILL_AFTER seconds, default 10), so the deadline always
|
|
# ends the provider before the caller's outer tool gate (#2776). Either
|
|
# way a deadline reports timeout(1)'s exit 124; output already written stays.
|
|
# An exported _CODEX_DEADLINE (epoch seconds) caps the duration at the time
|
|
# left; past it the command never starts.
|
|
local _duration="$1"
|
|
shift
|
|
local _grace="${_GSTACK_CODEX_KILL_AFTER:-10}"
|
|
local _to _rc _start _left
|
|
case "${_CODEX_DEADLINE:-}" in
|
|
''|*[!0-9]*) ;;
|
|
*) _left=$(( _CODEX_DEADLINE - $(date +%s 2>/dev/null || echo 0) ))
|
|
if [ "$_left" -le 0 ]; then
|
|
echo "gstack: the invocation's deadline passed before this command started; nothing ran." >&2
|
|
return 124
|
|
fi
|
|
_start="${_duration%%.*}"
|
|
[ "${_start:-0}" -ge "$_left" ] && _duration="$_left" ;;
|
|
esac
|
|
_to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "")
|
|
if [ -n "$_to" ]; then
|
|
_start=$(date +%s 2>/dev/null || echo 0)
|
|
"$_to" -k "$_grace" "$_duration" "$@"
|
|
_rc=$?
|
|
# 137 = timeout(1)'s KILL escalation; it is still the deadline firing.
|
|
if [ "$_rc" -eq 137 ] && [ $(( $(date +%s 2>/dev/null || echo 0) - _start )) -ge "${_duration%%.*}" ]; then
|
|
_rc=124
|
|
fi
|
|
return "$_rc"
|
|
else
|
|
# Stock macOS ships neither coreutils gtimeout nor timeout(1); running
|
|
# unwrapped let a hung `codex exec` block the probe — and the calling
|
|
# workflow — indefinitely. Emulate: background the command, TERM it at
|
|
# the deadline, KILL it after the grace period if it is still alive, and
|
|
# mirror timeout(1)'s exit-124 contract. The watchdog's stdout is
|
|
# detached so an early finish never blocks a caller's $(...) capture on
|
|
# the orphaned sleep.
|
|
# Without job control a background command's stdin is /dev/null; keep the
|
|
# caller's stdin so prompts fed with `codex exec -` arrive (#1674).
|
|
"$@" <&0 &
|
|
local _cmd_pid=$!
|
|
(
|
|
sleep "$_duration" && trap '' TERM && kill -TERM "$_cmd_pid" 2>/dev/null || exit 0 # best-effort: the command already exited, nothing to stop
|
|
_GSTACK_CODEX_WAITED=0
|
|
while [ "$_GSTACK_CODEX_WAITED" -lt "$_grace" ] && kill -0 "$_cmd_pid" 2>/dev/null; do
|
|
sleep 1
|
|
_GSTACK_CODEX_WAITED=$((_GSTACK_CODEX_WAITED + 1))
|
|
done
|
|
# Freeze the command first: killing its children while it runs lets it
|
|
# continue past the killed child and print output after its deadline.
|
|
kill -STOP "$_cmd_pid" 2>/dev/null
|
|
pkill -KILL -P "$_cmd_pid" 2>/dev/null
|
|
kill -KILL "$_cmd_pid" 2>/dev/null
|
|
exit 124
|
|
) >/dev/null 2>&1 &
|
|
local _watch_pid=$!
|
|
wait "$_cmd_pid"
|
|
_rc=$?
|
|
kill "$_watch_pid" 2>/dev/null
|
|
wait "$_watch_pid" 2>/dev/null
|
|
if [ "$?" -eq 124 ]; then
|
|
_rc=124
|
|
fi
|
|
return "$_rc"
|
|
fi
|
|
}
|
|
|
|
# --- Telemetry event --------------------------------------------------------
|
|
|
|
_gstack_codex_log_event() {
|
|
# Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl.
|
|
# Gated on $_TEL != "off" (caller sets this from gstack-config).
|
|
# Event types: codex_timeout, codex_auth_failed, codex_cli_missing,
|
|
# codex_version_warning, codex_model_unusable, codex_sandbox_unavailable,
|
|
# codex_model_retired, codex_cli_outdated, codex_quota_exhausted,
|
|
# codex_rate_limited.
|
|
# Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt
|
|
# content, env var values, or auth tokens.
|
|
local _event="$1"
|
|
local _duration="${2:-0}"
|
|
[ "${_TEL:-off}" = "off" ] && return 0
|
|
local _root
|
|
_root="$(. "$_GSTACK_CODEX_BIN/gstack-state-root.sh" 2>/dev/null && gstack_state_root)" || return 0
|
|
[ -n "$_root" ] || return 0
|
|
mkdir -p "$_root/analytics" 2>/dev/null || return 0
|
|
local _ts
|
|
_ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)
|
|
printf '{"skill":"codex","event":"%s","duration_s":"%s","ts":"%s"}\n' \
|
|
"$_event" "$_duration" "$_ts" \
|
|
>> "$_root/analytics/skill-usage.jsonl" 2>/dev/null || true
|
|
}
|
|
|
|
# --- Learnings log on hang --------------------------------------------------
|
|
|
|
_gstack_codex_log_hang() {
|
|
# Invoked when a codex invocation times out (exit 124). Records an
|
|
# operational learning so future /investigate sessions surface the pattern.
|
|
# Best-effort: errors swallowed.
|
|
local _mode="${1:-unknown}"
|
|
local _prompt_size="${2:-0}"
|
|
local _log_bin="$HOME/.claude/skills/gstack/bin/gstack-learnings-log"
|
|
[ -x "$_log_bin" ] || return 0
|
|
local _key="codex-hang-$(date +%s 2>/dev/null || echo unknown)"
|
|
"$_log_bin" "$(printf '{"skill":"codex","type":"operational","key":"%s","insight":"Codex hit its timeout wrapper during [%s] invocation. Prompt size: %s. Consider splitting prompt or checking network.","confidence":8,"source":"observed","files":["codex/SKILL.md.tmpl","autoplan/SKILL.md.tmpl"]}' "$_key" "$_mode" "$_prompt_size")" \
|
|
>/dev/null 2>&1 || true
|
|
}
|
|
|
|
# --- Executed subcommands ----------------------------------------------------
|
|
|
|
_gstack_codex_usage() {
|
|
printf '%s\n' \
|
|
'Usage: gstack-codex-probe <subcommand> [args]' \
|
|
' select-model <exec|review> [--model <id>] print CODEX_SEL:, CODEX_SEL_KIND: and CODEX_SANDBOX:; exit 0 or 1' \
|
|
' check-auth exit 0 when Codex credentials exist, 1 otherwise' \
|
|
' show-sandbox print CODEX_SANDBOX: read-only (or danger-full-access)' \
|
|
' check-sandbox free sandbox check; exit 0, or 3 with the reason on stderr' \
|
|
' probe-model <exec|review> [--model <id>] paid round trip; print CODEX_PROBE_STATE:; exit 0, 1, 2 or 4' \
|
|
' role-ready <exec|review> [--model <id>] [--cwd <dir>]' \
|
|
' plan-review selection, auth, sandbox and probe once; print' \
|
|
' CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:, CODEX_PROBE_STATE:' \
|
|
' check-version warn on stderr about known-bad Codex CLI versions; exit 0' \
|
|
' show-first-use-notice once per machine, say where reviews go; exit 0' \
|
|
' run-with-timeout <secs> <cmd...> run cmd under a deadline; its exit code, 124 on timeout' \
|
|
' log-event <event> [detail] record a telemetry event when telemetry is on; exit 0' \
|
|
' log-hang <skill> <prompt-size> record a timeout learning; exit 0' \
|
|
' help this text'
|
|
}
|
|
|
|
# Parses "<exec|review> [--model <id>]" into _GSTACK_CODEX_ARG_KIND and
|
|
# _GSTACK_CODEX_ARG_MODEL; returns 64 on anything else.
|
|
_gstack_codex_kind_args() {
|
|
_GSTACK_CODEX_ARG_KIND="${1:-}"
|
|
_GSTACK_CODEX_ARG_MODEL=""
|
|
case "$_GSTACK_CODEX_ARG_KIND" in exec|review) shift ;; *) return 64 ;; esac
|
|
if [ "$#" -eq 2 ] && [ "$1" = "--model" ] && [ -n "$2" ]; then
|
|
_GSTACK_CODEX_ARG_MODEL="$2"
|
|
elif [ "$#" -ne 0 ]; then
|
|
return 64
|
|
fi
|
|
}
|
|
|
|
# Parses "<exec|review> [--model <id>] [--cwd <dir>]" for role-ready.
|
|
_gstack_codex_role_args() {
|
|
_GSTACK_CODEX_ARG_KIND="${1:-}"
|
|
_GSTACK_CODEX_ARG_MODEL=""
|
|
_GSTACK_CODEX_ARG_CWD=""
|
|
case "$_GSTACK_CODEX_ARG_KIND" in exec|review) shift ;; *) return 64 ;; esac
|
|
while [ "$#" -gt 0 ]; do
|
|
{ [ "$#" -ge 2 ] && [ -n "$2" ]; } || return 64
|
|
case "$1" in
|
|
--model) _GSTACK_CODEX_ARG_MODEL="$2" ;;
|
|
--cwd) _GSTACK_CODEX_ARG_CWD="$2" ;;
|
|
*) return 64 ;;
|
|
esac
|
|
shift 2
|
|
done
|
|
}
|
|
|
|
_gstack_codex_main() {
|
|
local _cmd="${1:-}" _rc
|
|
[ "$#" -gt 0 ] && shift
|
|
case "$_cmd" in
|
|
log-event|probe-model|role-ready|check-sandbox|check-version)
|
|
# Telemetry is the caller's choice when it exports _TEL, else gstack-config's.
|
|
[ -n "${_TEL+set}" ] || _TEL=$("$_GSTACK_CODEX_BIN/gstack-config" get telemetry 2>/dev/null || echo off) ;;
|
|
esac
|
|
case "$_cmd" in
|
|
help|-h|--help) _gstack_codex_usage ;;
|
|
select-model)
|
|
_gstack_codex_kind_args "$@" || { _gstack_codex_usage >&2; return 64; }
|
|
_gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL" || return 1
|
|
printf 'CODEX_SEL: %s\nCODEX_SEL_KIND: %s\nCODEX_SANDBOX: %s\n' "$_GSTACK_CODEX_SEL" "$_GSTACK_CODEX_SEL_KIND" "$_GSTACK_CODEX_SANDBOX" ;;
|
|
check-auth) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_auth_probe ;;
|
|
show-sandbox)
|
|
[ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }
|
|
_gstack_codex_sandbox_mode
|
|
printf 'CODEX_SANDBOX: %s\n' "$_GSTACK_CODEX_SANDBOX" ;;
|
|
check-sandbox) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_sandbox_preflight >&2 ;;
|
|
probe-model)
|
|
_gstack_codex_kind_args "$@" || { _gstack_codex_usage >&2; return 64; }
|
|
if [ -n "$_GSTACK_CODEX_ARG_MODEL" ] && ! _gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL"; then
|
|
echo "MODEL_UNUSABLE"
|
|
echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
|
|
echo "CODEX_PROBE_STATE: unusable"
|
|
return 1
|
|
fi
|
|
_gstack_codex_model_probe "$_GSTACK_CODEX_ARG_KIND"
|
|
_rc=$?
|
|
case "$_rc" in
|
|
0) echo "CODEX_PROBE_STATE: ${_GSTACK_CODEX_PROBE_STATE:-ok}" ;;
|
|
1) echo "CODEX_PROBE_STATE: unusable" ;;
|
|
2) echo "CODEX_PROBE_STATE: broken_install" ;;
|
|
4) echo "CODEX_PROBE_STATE: quota_exhausted" ;;
|
|
*) echo "CODEX_PROBE_STATE: error" ;;
|
|
esac
|
|
return "$_rc" ;;
|
|
role-ready)
|
|
_gstack_codex_role_args "$@" || { _gstack_codex_usage >&2; return 64; }
|
|
_gstack_codex_role_ready "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_CWD" "$_GSTACK_CODEX_ARG_MODEL" >&2
|
|
_rc=$?
|
|
[ "$_rc" -eq 0 ] || return "$_rc"
|
|
printf 'CODEX_SEL: %s\nCODEX_SEL_KIND: %s\nCODEX_SANDBOX: %s\nCODEX_PROBE_STATE: %s\n' "$_GSTACK_CODEX_SEL" "$_GSTACK_CODEX_SEL_KIND" "$_GSTACK_CODEX_SANDBOX" "${_GSTACK_CODEX_PROBE_STATE:-ok}" ;;
|
|
check-version) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_version_check >&2; return 0 ;;
|
|
show-first-use-notice) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_first_use_notice; return 0 ;;
|
|
run-with-timeout)
|
|
case "${1:-}" in ''|*[!0-9.]*) _gstack_codex_usage >&2; return 64 ;; esac
|
|
[ "$#" -ge 2 ] || { _gstack_codex_usage >&2; return 64; }
|
|
_gstack_codex_timeout_wrapper "$@" ;;
|
|
log-event) [ "$#" -ge 1 ] && _gstack_codex_log_event "$@" >/dev/null 2>&1; return 0 ;;
|
|
log-hang) _gstack_codex_log_hang "$@" >/dev/null 2>&1; return 0 ;;
|
|
*) _gstack_codex_usage >&2; return 64 ;;
|
|
esac
|
|
}
|
|
|
|
# Executed (bash through the shebang, $0 names this file): run one subcommand.
|
|
# Sourced, including eval and zsh: keep the functions and say the form is going.
|
|
if [ -n "${BASH_VERSION:-}" ] && [ "${0##*/}" = "gstack-codex-probe" ]; then
|
|
_gstack_codex_main "$@"
|
|
exit $?
|
|
fi
|
|
printf '%s\n' "gstack: sourcing gstack-codex-probe is deprecated and stops working in a release on or after 2026-10-21. Run /gstack-upgrade so your skills run it as a command (gstack-codex-probe help)." >&2
|