From 3542322819e0aee6e56e65ae9f2e0efe9efda492 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 6 Oct 2026 20:42:35 +0000 Subject: [PATCH] fix: a Codex usage limit is quota_exhausted, a plain 429 is rate_limited, and only Codex's own error lines count (#3051 rework) The probe's quota signature used to treat any rate limit, "too many requests" or 429 as quota, and read the whole output, which echoes the prompt. Now only Codex's trailing ERROR: / stream error lines are read. A usage-limit or insufficient_quota line is MODEL_QUOTA_EXHAUSTED (exit 4, cached 15 minutes; quota wins over a 429 that names insufficient_quota); a plain 429 is MODEL_PROBE_RATE_LIMITED (exit 0, state rate_limited, never cached), reported as CODEX_MODE: unverified (rate_limited) so the review still runs and decides. The quota HINT says how many minutes gstack skips Codex and how to retry now (GSTACK_CODEX_PROBE_RETRY=1 skips the negative cache for that probe, or delete /.codex-model-probe), with the full troubleshooting URL. classifyOutsideReview reads the same trailing lines: a mid-review quota is unavailable (quota_exhausted), a mid-review 429 unavailable (rate_limited). New gate row rate_limited and docs anchor codex-rate-limited; the quota row documents the retry. Tests cover each line shape, the echoed prompt, exit 0, timeout, retry recovery, probe/classifier parity, and two fresh-shell preflight blocks where the second makes no paid Codex call. Co-authored-by: venkatareddyg --- autoplan/SKILL.md | 6 +- bin/gstack-codex-probe | 59 +++++++--- codex/SKILL.md | 10 +- codex/SKILL.md.tmpl | 10 +- docs/troubleshooting.md | 34 +++++- document-release/sections/release-body.md | 6 +- lib/gate-outcomes.ts | 9 +- lib/outside-review-result.ts | 27 ++++- plan-ceo-review/sections/review-sections.md | 6 +- plan-devex-review/sections/review-sections.md | 6 +- plan-eng-review/sections/review-sections.md | 6 +- review/sections/adversarial.md | 6 +- scripts/resolvers/constants.ts | 8 +- ship/sections/adversarial.md | 6 +- test/codex-model-probe.test.ts | 111 +++++++++++++++++- test/helpers/carve-guards.ts | 6 +- test/outside-review-result.test.ts | 69 +++++++++-- 17 files changed, 314 insertions(+), 71 deletions(-) diff --git a/autoplan/SKILL.md b/autoplan/SKILL.md index f168925ad..d47d4f3e0 100644 --- a/autoplan/SKILL.md +++ b/autoplan/SKILL.md @@ -784,6 +784,8 @@ else _CODEX_MODE="model_unusable" elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then _CODEX_MODE="unverified" + elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then + _CODEX_MODE="unverified (rate_limited)" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -799,9 +801,9 @@ Branch on the echoed `CODEX_MODE`: - **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=` or config.toml `model`); never substitute a model. Keep the required Claude adversarial pass; do not dispatch a duplicate. The ~10s round trip is cached for 1h. -- **`quota_exhausted`** — Codex usage/rate limit: relay its line verbatim (reset time), no more Codex calls. Keep the required Claude adversarial pass; do not dispatch a duplicate. +- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Keep the required Claude adversarial pass; do not dispatch a duplicate. -- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out; say so, and let the pass's own verdict decide. +- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide. Disabled/unavailable retains applicable native passes. Recheck each outside dispatch. Record provider and completed/unavailable/disabled/skipped per phase; CEO covers diff --git a/bin/gstack-codex-probe b/bin/gstack-codex-probe index 64a80c925..d642ede4a 100755 --- a/bin/gstack-codex-probe +++ b/bin/gstack-codex-probe @@ -9,7 +9,7 @@ # _gstack_codex_sandbox_preflight — free `codex sandbox` check before paid calls (Linux) # _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off # _gstack_codex_model_probe — round-trip probe of the selected model (#2477); -# exit 4 = the account's Codex usage limit (#3051) +# exit 4 = the account's Codex usage limit # _gstack_codex_version_check — warn on known-bad Codex CLI versions # _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog # _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/ @@ -224,11 +224,24 @@ _gstack_codex_model_probe() { # ENOENT, non-executable binary, missing vendor payload). Deterministic, # so fail-open is wrong: retrying never helps. Never cached — a reinstall # fixes it and must be picked up on the very next probe (#2742). + # MODEL_QUOTA_EXHAUSTED (exit 4) — Codex's usage-limit or insufficient_quota + # error on a failed, non-timeout call: the account, not the model. Codex's + # own line (with its reset time) is relayed verbatim; cached 15 min under + # the same signature as MODEL_UNUSABLE, so later fresh-shell blocks in the + # run make no Codex call. GSTACK_CODEX_PROBE_RETRY=1 skips both negative + # cache entries for that probe. + # MODEL_PROBE_RATE_LIMITED (exit 0) — a plain 429 / rate-limit error: often + # gone in seconds, so the review still runs, like the inconclusive case + # (_GSTACK_CODEX_PROBE_STATE=rate_limited, CODEX_MODE: unverified + # (rate_limited)). Codex's line is relayed verbatim; never cached. # MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; the review still # runs so a slow network never wedges codex mode, but readiness is # unverified (_GSTACK_CODEX_PROBE_STATE=inconclusive, CODEX_MODE: # unverified) and the review's own classification decides. Never cached. # + # Quota and rate-limit signatures read only Codex's own trailing error lines + # (ERROR: / stream error), never the whole output: it echoes the prompt. + # # Only call this AFTER _gstack_codex_auth_probe passes — probing without # auth just measures the auth failure again. local _kind="${1:-exec}" @@ -272,6 +285,7 @@ _gstack_codex_model_probe() { _sig=$(printf '%s\n' "$_model" "$_codex_home" "$_cfg_m" "$_auth_mode" "$_auth_m" "$_bin" "$_bin_m" | cksum | tr -d ' ') local _now _now=$(date +%s 2>/dev/null || echo 0) + local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md" if [ -n "$_cache" ] && [ -f "$_cache" ]; then local _c_line _c_status _c_ts _c_sig _c_line=$(head -1 "$_cache" 2>/dev/null) @@ -283,10 +297,11 @@ _gstack_codex_model_probe() { echo "MODEL_OK (cached)" return 0 fi + [ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry" if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then echo "MODEL_QUOTA_EXHAUSTED (cached)" sed -n '2,4p' "$_cache" 2>/dev/null - echo "HINT: Codex refused this account's calls for its usage limit within the last 15 minutes; outside coverage is unavailable until the reset time above. No Codex call was made." + echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted" return 4 fi if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then @@ -304,23 +319,38 @@ _gstack_codex_model_probe() { echo "MODEL_OK" return 0 fi - # A usage limit or rate limit is the account, not the model or the network - # (#3051): Codex says so plainly ("You've hit your usage limit ... try again - # at