feat(codex): model round-trip probe — an unusable configured model fails fast with guidance (#2477)

The auth probe accepts 'auth exists' as readiness, but a ChatGPT account
with a stale model pin in ~/.codex/config.toml passes it and then EVERY
mode dies with an HTTP 400 ('The <model> model is not supported when using
Codex with a ChatGPT account') and no pointer to where the model came from
— one report burned ~40 minutes and four invocations plus a strings dump
of the binary before finding the one-line config fix.

bin/gstack-codex-probe gains _gstack_codex_model_probe: a short
codex exec 'reply OK' round trip with the configured model, gated behind
the cheap auth probe at all three preflight sites (codex Step 0.5, the
shared codexPreflight in scripts/resolvers/constants.ts — which grows a
model_unusable CODEX_MODE branch — and autoplan's availability chain).
Verdicts: MODEL_OK (cached 1h, keyed on config.toml + auth.json mtimes so
a pin edit or re-login re-probes immediately), MODEL_UNUSABLE (exit 1,
prints the rejection plus HINTs at the model= pin and the
[notice.model_migrations] table), MODEL_PROBE_INCONCLUSIVE (timeout or
transient: FAIL-OPEN so network luck never wedges codex mode).

The 'Model not supported (HTTP 400)' Error Handling entry already shipped
in v1.64.0.0; Step 0.5's prose now routes MODEL_UNUSABLE to it.

test/codex-model-probe.test.ts drives all four behaviors against a stubbed
codex binary (invocation-counted cache hit, hint content, fail-open
polarity, mtime invalidation).

Fixes #2477

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

test/codex-model-probe.test.ts

@@ -0,0 +1,151 @@
+/**
+ * _gstack_codex_model_probe — round-trip model readiness (#2477).
+ *
+ * The auth probe accepts "auth exists" as readiness, but a ChatGPT account
+ * with a stale `model = "..."` pin in ~/.codex/config.toml passes auth and
+ * then dies with an HTTP 400 on every invocation. The model probe does one
+ * short `codex exec "reply OK"` round trip with the configured model.
+ *
+ * Contract pinned here (all runs use a STUBBED codex binary):
+ *   - exit 0            -> MODEL_OK, result cached (1h TTL + config/auth
+ *                          mtime signature), second call does NOT re-invoke
+ *   - model 400 output  -> MODEL_UNUSABLE (exit 1) + config.toml HINT lines,
+ *                          nothing cached
+ *   - transient failure -> MODEL_PROBE_INCONCLUSIVE, FAIL-OPEN (exit 0)
+ *   - config.toml mtime change invalidates a cached MODEL_OK
+ */
+import { describe, test, expect } from 'bun:test';
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
+
+const STUB = `#!/usr/bin/env bash
+echo "invoked" >> "$STUB_LOG"
+case "\${STUB_MODE:-ok}" in
+  ok) echo "OK"; exit 0 ;;
+  model400)
+    echo 'warning: Model metadata for \`gpt-5.4\` not found.' >&2
+    echo 'ERROR: {"type":"error","status":400,"error":{"type":"invalid_request_error","message":"The '"'"'gpt-5.4'"'"' model is not supported when using Codex with a ChatGPT account."}}' >&2
+    exit 1 ;;
+  transient) echo "stream error: network unreachable" >&2; exit 7 ;;
+esac
+`;
+
+interface Fixture {
+  home: string;
+  stubDir: string;
+  codexHome: string;
+  gstackHome: string;
+  stubLog: string;
+}
+
+function makeFixture(): Fixture {
+  const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-probe-'));
+  const stubDir = path.join(home, 'stub-bin');
+  const codexHome = path.join(home, '.codex');
+  const gstackHome = path.join(home, '.gstack');
+  fs.mkdirSync(stubDir, { recursive: true });
+  fs.mkdirSync(codexHome, { recursive: true });
+  fs.mkdirSync(gstackHome, { recursive: true });
+  fs.writeFileSync(path.join(stubDir, 'codex'), STUB, { mode: 0o755 });
+  fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n');
+  fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}');
+  const stubLog = path.join(home, 'stub.log');
+  return { home, stubDir, codexHome, gstackHome, stubLog };
+}
+
+function runProbe(f: Fixture, stubMode: string): { stdout: string; status: number } {
+  const result = spawnSync(
+    'bash',
+    ['-c', `set +e\nsource "${PROBE}"\n_gstack_codex_model_probe`],
+    {
+      env: {
+        PATH: `${f.stubDir}:${process.env.PATH ?? ''}`,
+        HOME: f.home,
+        CODEX_HOME: f.codexHome,
+        GSTACK_HOME: f.gstackHome,
+        STUB_MODE: stubMode,
+        STUB_LOG: f.stubLog,
+        _TEL: 'off',
+      },
+      timeout: 10000,
+    },
+  );
+  return { stdout: (result.stdout ?? '').toString(), status: result.status ?? -1 };
+}
+
+function invocations(f: Fixture): number {
+  try {
+    return fs.readFileSync(f.stubLog, 'utf-8').split('\n').filter(Boolean).length;
+  } catch {
+    return 0;
+  }
+}
+
+describe('codex model probe (#2477)', () => {
+  test('successful round trip -> MODEL_OK, cached, no re-invocation', () => {
+    const f = makeFixture();
+    try {
+      const first = runProbe(f, 'ok');
+      expect(first.stdout.trim()).toBe('MODEL_OK');
+      expect(first.status).toBe(0);
+      expect(invocations(f)).toBe(1);
+      expect(fs.existsSync(path.join(f.gstackHome, '.codex-model-probe'))).toBe(true);
+
+      const second = runProbe(f, 'ok');
+      expect(second.stdout.trim()).toBe('MODEL_OK (cached)');
+      expect(second.status).toBe(0);
+      expect(invocations(f)).toBe(1); // cache hit: stub not re-invoked
+    } finally {
+      fs.rmSync(f.home, { recursive: true, force: true });
+    }
+  });
+
+  test('model 400 -> MODEL_UNUSABLE with config.toml hints, exit 1, nothing cached', () => {
+    const f = makeFixture();
+    try {
+      const r = runProbe(f, 'model400');
+      expect(r.stdout).toContain('MODEL_UNUSABLE');
+      expect(r.stdout).toContain('config.toml');
+      expect(r.stdout).toContain('model_migrations');
+      // Surfaces the actual rejection so the user sees WHICH model.
+      expect(r.stdout).toContain('gpt-5.4');
+      expect(r.status).toBe(1);
+      expect(fs.existsSync(path.join(f.gstackHome, '.codex-model-probe'))).toBe(false);
+    } finally {
+      fs.rmSync(f.home, { recursive: true, force: true });
+    }
+  });
+
+  test('transient failure -> inconclusive, FAIL-OPEN exit 0', () => {
+    const f = makeFixture();
+    try {
+      const r = runProbe(f, 'transient');
+      expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE');
+      expect(r.status).toBe(0);
+    } finally {
+      fs.rmSync(f.home, { recursive: true, force: true });
+    }
+  });
+
+  test('config.toml change invalidates the cached MODEL_OK', () => {
+    const f = makeFixture();
+    try {
+      runProbe(f, 'ok');
+      expect(invocations(f)).toBe(1);
+      // Change the model pin; mtime signature must invalidate the cache.
+      fs.writeFileSync(path.join(f.codexHome, 'config.toml'), 'model = "gpt-5.5"\n');
+      const future = Date.now() / 1000 + 10;
+      fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future);
+      const r = runProbe(f, 'ok');
+      expect(r.stdout.trim()).toBe('MODEL_OK');
+      expect(invocations(f)).toBe(2); // re-probed
+    } finally {
+      fs.rmSync(f.home, { recursive: true, force: true });
+    }
+  });
+});

test/fixtures/golden/factory-ship-SKILL.md

@@ -2501,6 +2501,8 @@ elif ! command -v codex >/dev/null 2>&1; then
   _CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
 elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
   _CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
+elif ! _gstack_codex_model_probe; then
+  _CODEX_MODE="model_unusable"
 else
   _CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
 fi
@@ -2511,6 +2513,7 @@ Branch on the echoed `CODEX_MODE`:
 - **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only."
 - **`not_installed`** — Codex CLI absent. Print: "Codex not installed — using Claude subagent. Install for cross-model coverage: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
 - **`not_authed`** — installed but no credentials. Print: "Codex installed but not authenticated — using Claude subagent. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
+- **`model_unusable`** — authed but the account cannot use its configured model (#2477: HTTP 400 on every call, usually a stale `model =` pin in `~/.codex/config.toml`). Relay the probe's HINT lines, tell the user the one-line fix (update the pin; `[notice.model_migrations]` names the replacement), and fall back to the Claude subagent path. The ~10s round trip is cached for 1h; timeouts fail open to `ready`.
 - **`ready`** — run the Codex pass below.
 
 For this diff-review path, `CODEX_MODE: disabled` means skip the Codex passes ONLY — the