From ee70becad52249e797db83b770fbadcf586fe178 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 21:05:23 +0000 Subject: [PATCH] B2: a retired-model 404 reports MODEL_UNUSABLE, not CODEX_MODE: ready The model probe matched only 400 / 'is not supported', so a retired model's 404 ('does not exist or you do not have access to it') and a custom provider's bare 404 fell through to the inconclusive branch and reported ready. Both now classify as MODEL_UNUSABLE with a hint naming the two causes (retired model, or a wrong base_url) and log codex_model_retired; a 'requires a newer version of Codex' 400 points at upgrading the CLI and logs codex_cli_outdated. Ported from #2843 onto the current probe. Co-authored-by: Marek Cermak --- bin/gstack-codex-probe | 30 +++++++++++++++++++++++------- codex/SKILL.md.tmpl | 13 ++++++++----- test/codex-model-probe.test.ts | 27 +++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/bin/gstack-codex-probe b/bin/gstack-codex-probe index 6b09ee254..96c640b7f 100755 --- a/bin/gstack-codex-probe +++ b/bin/gstack-codex-probe @@ -118,8 +118,9 @@ _gstack_codex_model_probe() { # # Contract: # MODEL_OK (exit 0) — round trip succeeded; cached 1h. - # MODEL_UNUSABLE (exit 1) — deterministic model 400 or an invalid - # model choice; hints printed. The 400 is cached 15 min: re-probing every + # MODEL_UNUSABLE (exit 1) — deterministic model rejection (400, or a + # 404 from a retired model or a wrong custom-provider base_url) or an + # invalid model choice; hints name the cause. The 400 is cached 15 min: re-probing every # preflight charged the affected user a 30s round trip + real tokens per # review section, forever. Changing the model or config.toml changes the # cache signature and re-probes immediately; the short TTL covers @@ -202,13 +203,27 @@ _gstack_codex_model_probe() { echo "MODEL_OK" return 0 fi - if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*400'; then + # A model rejection is deterministic: a 400 ("is not supported", "requires a + # newer version of Codex") or a 404. A retired model answers 404 "does not + # exist or you do not have access to it", and a custom provider with a wrong + # base_url answers a bare 404; neither is network luck, so neither may fall + # through to the inconclusive branch below and report ready (#2843). + if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; } echo "MODEL_UNUSABLE" - printf '%s\n' "$_out" | grep -i "model" | head -3 + printf '%s\n' "$_out" | grep -iE "model|404" | head -3 echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." - echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." - _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true + if printf '%s' "$_out" | grep -qiE 'requires a newer version of codex|upgrade to the latest (app|version|cli)'; then + echo "HINT: the server refused it because this Codex CLI is too old; no model choice fixes that. Upgrade the CLI (\`codex doctor\` prints the command; usually npm install -g @openai/codex)." + _gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true + elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then + echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong." + echo "HINT: choose a current model (GSTACK_CODEX_MODEL= or model in config.toml), or fix that provider's base_url." + _gstack_codex_log_event "codex_model_retired" 2>/dev/null || true + else + echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." + _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true + fi return 1 fi # A CLI that cannot execute is deterministic, not transient: the fail-open @@ -331,7 +346,8 @@ _gstack_codex_log_event() { # Emit a telemetry event to ~/.gstack/analytics/skill-usage.jsonl. # Gated on $_TEL != "off" (caller sets this from gstack-config). # Event types: codex_timeout, codex_auth_failed, codex_cli_missing, - # codex_version_warning, codex_model_unusable, codex_sandbox_unavailable. + # codex_version_warning, codex_model_unusable, codex_sandbox_unavailable, +# codex_model_retired, codex_cli_outdated. # Payload schema: {skill, event, duration_s, ts}. NEVER includes prompt # content, env var values, or auth tokens. local _event="$1" diff --git a/codex/SKILL.md.tmpl b/codex/SKILL.md.tmpl index 69b730ced..f132338b9 100644 --- a/codex/SKILL.md.tmpl +++ b/codex/SKILL.md.tmpl @@ -97,7 +97,7 @@ If the output contains `AUTH_FAILED`, stop and tell the user: If the output contains `MODEL_UNUSABLE`, stop — the selected model (named with its source on the `CODEX_MODEL:` line) is invalid or the account cannot use it. Relay the probe's HINT lines and -follow the "Model not supported (HTTP 400)" recovery steps in +follow the "Model not supported (HTTP 400 or 404)" recovery steps in `## Error Handling` below. Running the modes anyway just burns four invocations on the same 400 (#2477). @@ -293,11 +293,14 @@ If token count is not available, display: `Tokens: unknown` missing or wrong. A prompt-only `codex review` defaults to uncommitted changes, so a clean working tree reads as an empty review even when `...HEAD` is large. Confirm `--base ` is actually on the command line. -- **Model not supported (HTTP 400):** stderr shows +- **Model not supported (HTTP 400 or 404):** stderr shows `The '' model is not supported when using Codex with a ChatGPT account` - (a `status: 400` / `invalid_request_error` naming a model). This is a - model-entitlement problem, not an auth or network failure, and the auth probe - cannot catch it. Recovery, in order: + (a `status: 400` / `invalid_request_error` naming a model), or + `404 Not Found: The model '' does not exist or you do not have access to it` + (a retired model; a bare 404 usually means a custom provider's `base_url` is wrong). + `requires a newer version of Codex` means the CLI is too old: upgrade it instead. + None of these is an auth or network failure, and the auth probe cannot catch them. + Recovery, in order: 1. Read the `CODEX_MODEL:` line: it names the model and where it came from. 2. Fix that source: name another model for this request, update `GSTACK_CODEX_MODEL`, or change `model` (or `review_model`) in the Codex diff --git a/test/codex-model-probe.test.ts b/test/codex-model-probe.test.ts index 1f7d3621b..952bb8567 100644 --- a/test/codex-model-probe.test.ts +++ b/test/codex-model-probe.test.ts @@ -53,6 +53,13 @@ case "\${STUB_MODE:-ok}" in echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-6-astra'"'"' model is not supported when using Codex with a ChatGPT account."}}' >&2 exit 1 ;; transient) echo "stream error: network unreachable" >&2; exit 7 ;; + retired404) + echo 'ERROR: unexpected status 404 Not Found: The model \`gpt-5.2-codex\` does not exist or you do not have access to it., url: https://chatgpt.com/backend-api/codex/responses' >&2 + exit 1 ;; + baseurl404) echo 'ERROR: unexpected status 404 Not Found: Not Found, url: http://localhost:9999/v1/responses' >&2; exit 1 ;; + outdated400) + echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-6-astra'"'"' model requires a newer version of Codex. Please upgrade to the latest app or CLI and try again."}}' >&2 + exit 1 ;; esac `; @@ -446,3 +453,23 @@ describe('B1: codex sandbox preflight and unverified readiness', () => { } }); }); + +describe('B2: a 404 model rejection is unusable, never ready (#2843)', () => { + for (const [mode, hint, event] of [ + ['retired404', "Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url", 'codex_model_retired'], + ['baseurl404', "a custom provider's base_url", 'codex_model_retired'], + ['outdated400', 'this Codex CLI is too old', 'codex_cli_outdated'], + ] as const) { + test(`${mode} -> MODEL_UNUSABLE with a hint naming the cause and a ${event} event`, () => { + const f = makeFixture(); + try { + const r = runProbe(f, mode, { _TEL: 'community' }, '_gstack_codex_model_probe; echo "rc=$?"'); + expect(r.stdout).toContain('MODEL_UNUSABLE'); + expect(r.stdout).toContain('rc=1'); + expect(r.stdout).not.toContain('MODEL_PROBE_INCONCLUSIVE'); + expect(r.stdout).toContain(hint); + expect(fs.readFileSync(path.join(f.gstackHome, 'analytics', 'skill-usage.jsonl'), 'utf-8')).toContain(`"event":"${event}"`); + } finally { fs.rmSync(f.home, { recursive: true, force: true }); } + }); + } +});