From 73c96f9c129c462621ecac228b19b8396d35f1b9 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 16 Aug 2026 10:41:12 -0700 Subject: [PATCH] =?UTF-8?q?feat(codex):=20model=20round-trip=20probe=20?= =?UTF-8?q?=E2=80=94=20an=20unusable=20configured=20model=20fails=20fast?= =?UTF-8?q?=20with=20guidance=20(#2477)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auth probe accepts 'auth exists' as readiness, but a ChatGPT account with a stale model pin in ~/.codex/config.toml passes it and then EVERY mode dies with an HTTP 400 ('The model is not supported when using Codex with a ChatGPT account') and no pointer to where the model came from — one report burned ~40 minutes and four invocations plus a strings dump of the binary before finding the one-line config fix. bin/gstack-codex-probe gains _gstack_codex_model_probe: a short codex exec 'reply OK' round trip with the configured model, gated behind the cheap auth probe at all three preflight sites (codex Step 0.5, the shared codexPreflight in scripts/resolvers/constants.ts — which grows a model_unusable CODEX_MODE branch — and autoplan's availability chain). Verdicts: MODEL_OK (cached 1h, keyed on config.toml + auth.json mtimes so a pin edit or re-login re-probes immediately), MODEL_UNUSABLE (exit 1, prints the rejection plus HINTs at the model= pin and the [notice.model_migrations] table), MODEL_PROBE_INCONCLUSIVE (timeout or transient: FAIL-OPEN so network luck never wedges codex mode). The 'Model not supported (HTTP 400)' Error Handling entry already shipped in v1.64.0.0; Step 0.5's prose now routes MODEL_UNUSABLE to it. test/codex-model-probe.test.ts drives all four behaviors against a stubbed codex binary (invocation-counted cache hit, hint content, fail-open polarity, mtime invalidation). Fixes #2477 Co-Authored-By: Claude Fable 5 --- autoplan/SKILL.md | 6 + autoplan/SKILL.md.tmpl | 6 + bin/gstack-codex-probe | 69 +++++++- codex/SKILL.md | 21 ++- codex/SKILL.md.tmpl | 21 ++- document-release/sections/release-body.md | 3 + plan-ceo-review/sections/review-sections.md | 3 + plan-devex-review/sections/review-sections.md | 3 + plan-eng-review/sections/review-sections.md | 3 + review/SKILL.md | 3 + scripts/resolvers/constants.ts | 3 + ship/sections/adversarial.md | 3 + test/codex-model-probe.test.ts | 151 ++++++++++++++++++ test/fixtures/golden/factory-ship-SKILL.md | 3 + 14 files changed, 289 insertions(+), 9 deletions(-) create mode 100644 test/codex-model-probe.test.ts diff --git a/autoplan/SKILL.md b/autoplan/SKILL.md index 3aa4b9e9c..6324f5e2b 100644 --- a/autoplan/SKILL.md +++ b/autoplan/SKILL.md @@ -1157,6 +1157,12 @@ elif ! _gstack_codex_auth_probe >/dev/null; then _gstack_codex_log_event "codex_auth_failed" echo "[codex-unavailable: auth missing] — proceeding with Claude subagent only. Run \`codex login\` or set \$CODEX_API_KEY to enable dual-voice review." _CODEX_AVAILABLE=false +# Round-trip model probe (#2477): auth can pass while the account's configured +# model is rejected with an HTTP 400 (stale `model =` pin in ~/.codex/config.toml). +# ~10s on first run, cached 1h; timeouts fail open (probe returns 0). +elif ! _gstack_codex_model_probe; then + echo "[codex-unavailable: configured model rejected] — proceeding with Claude subagent only. Fix the \`model =\` pin in ~/.codex/config.toml (see [notice.model_migrations] there for the replacement)." + _CODEX_AVAILABLE=false else _gstack_codex_version_check # non-blocking warn if known-bad _CODEX_AVAILABLE=true diff --git a/autoplan/SKILL.md.tmpl b/autoplan/SKILL.md.tmpl index 1b3d183f9..08d7933e9 100644 --- a/autoplan/SKILL.md.tmpl +++ b/autoplan/SKILL.md.tmpl @@ -262,6 +262,12 @@ elif ! _gstack_codex_auth_probe >/dev/null; then _gstack_codex_log_event "codex_auth_failed" echo "[codex-unavailable: auth missing] — proceeding with Claude subagent only. Run \`codex login\` or set \$CODEX_API_KEY to enable dual-voice review." _CODEX_AVAILABLE=false +# Round-trip model probe (#2477): auth can pass while the account's configured +# model is rejected with an HTTP 400 (stale `model =` pin in ~/.codex/config.toml). +# ~10s on first run, cached 1h; timeouts fail open (probe returns 0). +elif ! _gstack_codex_model_probe; then + echo "[codex-unavailable: configured model rejected] — proceeding with Claude subagent only. Fix the \`model =\` pin in ~/.codex/config.toml (see [notice.model_migrations] there for the replacement)." + _CODEX_AVAILABLE=false else _gstack_codex_version_check # non-blocking warn if known-bad _CODEX_AVAILABLE=true diff --git a/bin/gstack-codex-probe b/bin/gstack-codex-probe index 940dacf84..47f8a68cb 100755 --- a/bin/gstack-codex-probe +++ b/bin/gstack-codex-probe @@ -4,6 +4,7 @@ # # Functions (all prefixed with _gstack_codex_ for namespace hygiene): # _gstack_codex_auth_probe — multi-signal auth check (env + file) +# _gstack_codex_model_probe — round-trip probe of the configured model (#2477) # _gstack_codex_version_check — warn on known-bad Codex CLI versions # _gstack_codex_timeout_wrapper — gtimeout -> timeout -> unwrapped fallback # _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/ @@ -33,6 +34,72 @@ _gstack_codex_auth_probe() { return 1 } +# --- Model round-trip probe (#2477) ------------------------------------------ + +_gstack_codex_model_probe() { + # Auth-exists is a weaker signal than the auth probe implies: a ChatGPT + # account with a stale `model = "..."` pin in ~/.codex/config.toml passes + # the auth probe, then EVERY invocation dies with an HTTP 400 ("The + # '' model is not supported when using Codex with a ChatGPT + # account") and no guidance. A short real round trip with the configured + # model catches model rejection, entitlement changes, and stale pins in + # one shot (#2477). + # + # Contract: + # MODEL_OK (exit 0) — round trip succeeded; cached 1h. + # MODEL_UNUSABLE (exit 1) — deterministic model 400; hints printed. + # MODEL_PROBE_INCONCLUSIVE (exit 0) — timeout/transient; FAIL-OPEN so a + # slow network never wedges codex mode (the per-invocation Error + # Handling entry still covers a later 400). + # + # Only call this AFTER _gstack_codex_auth_probe passes — probing without + # auth just measures the auth failure again. + local _codex_home="${CODEX_HOME:-$HOME/.codex}" + local _gstack_home="${GSTACK_HOME:-$HOME/.gstack}" + local _cache="$_gstack_home/.codex-model-probe" + # Cache signature: config.toml + auth.json mtimes. Editing the model pin + # or re-logging-in invalidates the cached MODEL_OK immediately. + local _cfg_m _auth_m _sig + _cfg_m=$(stat -f %m "$_codex_home/config.toml" 2>/dev/null || stat -c %Y "$_codex_home/config.toml" 2>/dev/null || echo 0) + _auth_m=$(stat -f %m "$_codex_home/auth.json" 2>/dev/null || stat -c %Y "$_codex_home/auth.json" 2>/dev/null || echo 0) + _sig="${_cfg_m}-${_auth_m}" + local _now + _now=$(date +%s 2>/dev/null || echo 0) + if [ -f "$_cache" ]; then + local _c_line _c_status _c_ts _c_sig + _c_line=$(head -1 "$_cache" 2>/dev/null) + _c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1) + _c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2) + _c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3) + case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac + if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then + echo "MODEL_OK (cached)" + return 0 + fi + fi + local _out _code + _out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only "reply OK" &1) + _code=$? + if [ "$_code" -eq 0 ]; then + mkdir -p "$_gstack_home" 2>/dev/null || true + printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null || true + echo "MODEL_OK" + return 0 + fi + if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*400'; then + echo "MODEL_UNUSABLE" + printf '%s\n' "$_out" | grep -i "model" | head -3 + echo "HINT: the rejected model comes from the 'model = ' line in $_codex_home/config.toml." + echo "HINT: check its [notice.model_migrations] table — Codex records the intended replacement there." + _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true + return 1 + fi + # Timeout (124) or transient failure: fail-open with a warning. The probe + # exists to catch the deterministic model 400, not to gate on network luck. + echo "MODEL_PROBE_INCONCLUSIVE (exit $_code) — proceeding; if invocations fail with a model 400, see the codex skill's Error Handling entry." + return 0 +} + # --- Version check ---------------------------------------------------------- _gstack_codex_version_check() { @@ -72,7 +139,7 @@ _gstack_codex_log_event() { # Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl. # Gated on $_TEL != "off" (caller sets this from gstack-config). # Event types: codex_timeout, codex_auth_failed, codex_cli_missing, - # codex_version_warning. + # codex_version_warning, codex_model_unusable. # Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt # content, env var values, or auth tokens. local _event="$1" diff --git a/codex/SKILL.md b/codex/SKILL.md index 006cea28d..8d0b40261 100644 --- a/codex/SKILL.md +++ b/codex/SKILL.md @@ -884,11 +884,12 @@ source ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null && _gstack_cod --- -## Step 0.5: Auth probe + version check +## Step 0.5: Auth probe + model probe + version check -Before building expensive prompts, verify Codex has valid auth AND the installed -CLI version isn't in the known-bad list. Sourcing `gstack-codex-probe` loads the -shared helpers that both `/codex` and `/autoplan` use. +Before building expensive prompts, verify Codex has valid auth, that the account +can actually USE its configured model, AND the installed CLI version isn't in the +known-bad list. Sourcing `gstack-codex-probe` loads the shared helpers that both +`/codex` and `/autoplan` use. ```bash _TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off) @@ -897,6 +898,8 @@ source ~/.claude/skills/gstack/bin/gstack-codex-probe if ! _gstack_codex_auth_probe >/dev/null; then _gstack_codex_log_event "codex_auth_failed" echo "AUTH_FAILED" +else + _gstack_codex_model_probe # ~10s round trip on first run, cached 1h (#2477) fi _gstack_codex_version_check # warns if known-bad, non-blocking ``` @@ -904,6 +907,16 @@ _gstack_codex_version_check # warns if known-bad, non-blocking If the output contains `AUTH_FAILED`, stop and tell the user: "No Codex authentication found. Run `codex login` or set `$CODEX_API_KEY` / `$OPENAI_API_KEY`, then re-run this skill." +If the output contains `MODEL_UNUSABLE`, stop — auth exists but the account +cannot use the configured model (a stale `model =` pin in +`~/.codex/config.toml` is the usual cause). Relay the probe's HINT lines and +follow the "Model not supported (HTTP 400)" recovery steps in +`## Error Handling` below. Running the modes anyway just burns four +invocations on the same 400 (#2477). + +`MODEL_PROBE_INCONCLUSIVE` is non-blocking (timeout/transient network): pass +the warning through and continue. + If the version check printed a `WARN:` line, pass it through to the user verbatim (non-blocking — Codex may still work, but the user should upgrade). diff --git a/codex/SKILL.md.tmpl b/codex/SKILL.md.tmpl index 88f48d00a..bb54dca12 100644 --- a/codex/SKILL.md.tmpl +++ b/codex/SKILL.md.tmpl @@ -57,11 +57,12 @@ source ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null && _gstack_cod --- -## Step 0.5: Auth probe + version check +## Step 0.5: Auth probe + model probe + version check -Before building expensive prompts, verify Codex has valid auth AND the installed -CLI version isn't in the known-bad list. Sourcing `gstack-codex-probe` loads the -shared helpers that both `/codex` and `/autoplan` use. +Before building expensive prompts, verify Codex has valid auth, that the account +can actually USE its configured model, AND the installed CLI version isn't in the +known-bad list. Sourcing `gstack-codex-probe` loads the shared helpers that both +`/codex` and `/autoplan` use. ```bash _TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off) @@ -70,6 +71,8 @@ source ~/.claude/skills/gstack/bin/gstack-codex-probe if ! _gstack_codex_auth_probe >/dev/null; then _gstack_codex_log_event "codex_auth_failed" echo "AUTH_FAILED" +else + _gstack_codex_model_probe # ~10s round trip on first run, cached 1h (#2477) fi _gstack_codex_version_check # warns if known-bad, non-blocking ``` @@ -77,6 +80,16 @@ _gstack_codex_version_check # warns if known-bad, non-blocking If the output contains `AUTH_FAILED`, stop and tell the user: "No Codex authentication found. Run `codex login` or set `$CODEX_API_KEY` / `$OPENAI_API_KEY`, then re-run this skill." +If the output contains `MODEL_UNUSABLE`, stop — auth exists but the account +cannot use the configured model (a stale `model =` pin in +`~/.codex/config.toml` is the usual cause). Relay the probe's HINT lines and +follow the "Model not supported (HTTP 400)" recovery steps in +`## Error Handling` below. Running the modes anyway just burns four +invocations on the same 400 (#2477). + +`MODEL_PROBE_INCONCLUSIVE` is non-blocking (timeout/transient network): pass +the warning through and continue. + If the version check printed a `WARN:` line, pass it through to the user verbatim (non-blocking — Codex may still work, but the user should upgrade). diff --git a/document-release/sections/release-body.md b/document-release/sections/release-body.md index 6efdaf259..f240f4d25 100644 --- a/document-release/sections/release-body.md +++ b/document-release/sections/release-body.md @@ -381,6 +381,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -391,6 +393,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. When the mode is `ready`, `not_installed`, or `not_authed`, print one line so the off-switch diff --git a/plan-ceo-review/sections/review-sections.md b/plan-ceo-review/sections/review-sections.md index bed5c088e..55c51c95c 100644 --- a/plan-ceo-review/sections/review-sections.md +++ b/plan-ceo-review/sections/review-sections.md @@ -274,6 +274,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -284,6 +286,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. When the mode is `ready`, `not_installed`, or `not_authed`, print one line so the off-switch diff --git a/plan-devex-review/sections/review-sections.md b/plan-devex-review/sections/review-sections.md index 82e79e7f7..4e65ed858 100644 --- a/plan-devex-review/sections/review-sections.md +++ b/plan-devex-review/sections/review-sections.md @@ -260,6 +260,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -270,6 +272,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. When the mode is `ready`, `not_installed`, or `not_authed`, print one line so the off-switch diff --git a/plan-eng-review/sections/review-sections.md b/plan-eng-review/sections/review-sections.md index e1c252a54..cb60b5015 100644 --- a/plan-eng-review/sections/review-sections.md +++ b/plan-eng-review/sections/review-sections.md @@ -355,6 +355,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -365,6 +367,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. When the mode is `ready`, `not_installed`, or `not_authed`, print one line so the off-switch diff --git a/review/SKILL.md b/review/SKILL.md index c8d2140d3..e6aec2d17 100644 --- a/review/SKILL.md +++ b/review/SKILL.md @@ -1685,6 +1685,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -1695,6 +1697,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. For this diff-review path, `CODEX_MODE: disabled` means skip the Codex passes ONLY — the diff --git a/scripts/resolvers/constants.ts b/scripts/resolvers/constants.ts index c665de9f4..f0e0c9eaa 100644 --- a/scripts/resolvers/constants.ts +++ b/scripts/resolvers/constants.ts @@ -122,6 +122,8 @@ elif ! command -v codex >/dev/null 2>&1; then ${m}="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then ${m}="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + ${m}="model_unusable" else ${m}="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -132,5 +134,6 @@ Branch on the echoed \`CODEX_MODE\`: - **\`disabled\`** — the user turned Codex reviews off (\`codex_reviews=disabled\`). ${disabledLine} - **\`not_installed\`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: \`npm install -g @openai/codex\`." Fall back to the Claude subagent path. - **\`not_authed\`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run \`codex login\` or set \`$CODEX_API_KEY\`." Fall back to the Claude subagent path. +- **\`model_unusable\`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale \`model =\` pin in \`~/.codex/config.toml\`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; \`[notice.model_migrations]\` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to \`ready\`. - **\`ready\`** — run the Codex pass below.`; } diff --git a/ship/sections/adversarial.md b/ship/sections/adversarial.md index f4f9410df..8cf99a5b4 100644 --- a/ship/sections/adversarial.md +++ b/ship/sections/adversarial.md @@ -27,6 +27,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -37,6 +39,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. For this diff-review path, `CODEX_MODE: disabled` means skip the Codex passes ONLY — the diff --git a/test/codex-model-probe.test.ts b/test/codex-model-probe.test.ts new file mode 100644 index 000000000..2915fd01c --- /dev/null +++ b/test/codex-model-probe.test.ts @@ -0,0 +1,151 @@ +/** + * _gstack_codex_model_probe — round-trip model readiness (#2477). + * + * The auth probe accepts "auth exists" as readiness, but a ChatGPT account + * with a stale `model = "..."` pin in ~/.codex/config.toml passes auth and + * then dies with an HTTP 400 on every invocation. The model probe does one + * short `codex exec "reply OK"` round trip with the configured model. + * + * Contract pinned here (all runs use a STUBBED codex binary): + * - exit 0 -> MODEL_OK, result cached (1h TTL + config/auth + * mtime signature), second call does NOT re-invoke + * - model 400 output -> MODEL_UNUSABLE (exit 1) + config.toml HINT lines, + * nothing cached + * - transient failure -> MODEL_PROBE_INCONCLUSIVE, FAIL-OPEN (exit 0) + * - config.toml mtime change invalidates a cached MODEL_OK + */ +import { describe, test, expect } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe'); + +const STUB = `#!/usr/bin/env bash +echo "invoked" >> "$STUB_LOG" +case "\${STUB_MODE:-ok}" in + ok) echo "OK"; exit 0 ;; + model400) + echo 'warning: Model metadata for \`gpt-5.4\` not found.' >&2 + echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-5.4'"'"' model is not supported when using Codex with a ChatGPT account."}}' >&2 + exit 1 ;; + transient) echo "stream error: network unreachable" >&2; exit 7 ;; +esac +`; + +interface Fixture { + home: string; + stubDir: string; + codexHome: string; + gstackHome: string; + stubLog: string; +} + +function makeFixture(): Fixture { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-probe-')); + const stubDir = path.join(home, 'stub-bin'); + const codexHome = path.join(home, '.codex'); + const gstackHome = path.join(home, '.gstack'); + fs.mkdirSync(stubDir, { recursive: true }); + fs.mkdirSync(codexHome, { recursive: true }); + fs.mkdirSync(gstackHome, { recursive: true }); + fs.writeFileSync(path.join(stubDir, 'codex'), STUB, { mode: 0o755 }); + fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n'); + fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}'); + const stubLog = path.join(home, 'stub.log'); + return { home, stubDir, codexHome, gstackHome, stubLog }; +} + +function runProbe(f: Fixture, stubMode: string): { stdout: string; status: number } { + const result = spawnSync( + 'bash', + ['-c', `set +e\nsource "${PROBE}"\n_gstack_codex_model_probe`], + { + env: { + PATH: `${f.stubDir}:${process.env.PATH ?? ''}`, + HOME: f.home, + CODEX_HOME: f.codexHome, + GSTACK_HOME: f.gstackHome, + STUB_MODE: stubMode, + STUB_LOG: f.stubLog, + _TEL: 'off', + }, + timeout: 10000, + }, + ); + return { stdout: (result.stdout ?? '').toString(), status: result.status ?? -1 }; +} + +function invocations(f: Fixture): number { + try { + return fs.readFileSync(f.stubLog, 'utf-8').split('\n').filter(Boolean).length; + } catch { + return 0; + } +} + +describe('codex model probe (#2477)', () => { + test('successful round trip -> MODEL_OK, cached, no re-invocation', () => { + const f = makeFixture(); + try { + const first = runProbe(f, 'ok'); + expect(first.stdout.trim()).toBe('MODEL_OK'); + expect(first.status).toBe(0); + expect(invocations(f)).toBe(1); + expect(fs.existsSync(path.join(f.gstackHome, '.codex-model-probe'))).toBe(true); + + const second = runProbe(f, 'ok'); + expect(second.stdout.trim()).toBe('MODEL_OK (cached)'); + expect(second.status).toBe(0); + expect(invocations(f)).toBe(1); // cache hit: stub not re-invoked + } finally { + fs.rmSync(f.home, { recursive: true, force: true }); + } + }); + + test('model 400 -> MODEL_UNUSABLE with config.toml hints, exit 1, nothing cached', () => { + const f = makeFixture(); + try { + const r = runProbe(f, 'model400'); + expect(r.stdout).toContain('MODEL_UNUSABLE'); + expect(r.stdout).toContain('config.toml'); + expect(r.stdout).toContain('model_migrations'); + // Surfaces the actual rejection so the user sees WHICH model. + expect(r.stdout).toContain('gpt-5.4'); + expect(r.status).toBe(1); + expect(fs.existsSync(path.join(f.gstackHome, '.codex-model-probe'))).toBe(false); + } finally { + fs.rmSync(f.home, { recursive: true, force: true }); + } + }); + + test('transient failure -> inconclusive, FAIL-OPEN exit 0', () => { + const f = makeFixture(); + try { + const r = runProbe(f, 'transient'); + expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE'); + expect(r.status).toBe(0); + } finally { + fs.rmSync(f.home, { recursive: true, force: true }); + } + }); + + test('config.toml change invalidates the cached MODEL_OK', () => { + const f = makeFixture(); + try { + runProbe(f, 'ok'); + expect(invocations(f)).toBe(1); + // Change the model pin; mtime signature must invalidate the cache. + fs.writeFileSync(path.join(f.codexHome, 'config.toml'), 'model = "gpt-5.5"\n'); + const future = Date.now() / 1000 + 10; + fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future); + const r = runProbe(f, 'ok'); + expect(r.stdout.trim()).toBe('MODEL_OK'); + expect(invocations(f)).toBe(2); // re-probed + } finally { + fs.rmSync(f.home, { recursive: true, force: true }); + } + }); +}); diff --git a/test/fixtures/golden/factory-ship-SKILL.md b/test/fixtures/golden/factory-ship-SKILL.md index de86452e0..66b1f8cb6 100644 --- a/test/fixtures/golden/factory-ship-SKILL.md +++ b/test/fixtures/golden/factory-ship-SKILL.md @@ -2501,6 +2501,8 @@ elif ! command -v codex >/dev/null 2>&1; then _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true +elif ! _gstack_codex_model_probe; then + _CODEX_MODE="model_unusable" else _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true fi @@ -2511,6 +2513,7 @@ Branch on the echoed `CODEX_MODE`: - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only." - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path. - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path. +- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`. - **`ready`** — run the Codex pass below. For this diff-review path, `CODEX_MODE: disabled` means skip the Codex passes ONLY — the