fix: preserve actionable model failures and validate review inputs before probes

bin/gstack-config

@@ -152,18 +152,16 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne
 #                           #   returns to the generic overlay.
 #
 # ─── Model policy (gstack-models shows the effective choices) ───────
-# plan_review_tier: frontier  # frontier | smart | host — model tier for the
-#                           #   opposing-provider plan review. host = the
-#                           #   Codex / Claude Code CLI settings choose.
-#                           #   GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL win.
-# implementation_tier: smart  # frontier | smart — tier gstack recommends for
-#                           #   implementing an approved plan (never switches
-#                           #   your session).
-# model_frontier_claude:    # per-tier model IDs; unset = the shipped catalog
-# model_frontier_openai:    #   recommendation (gstack-models list). Invalid
-# model_smart_claude:       #   values are REJECTED and the stored value kept;
-# model_smart_openai:       #   unset restores the shipped recommendation.
-#                           # Docs: docs/model-policy.md
+# Plan review tier: frontier | smart | host. host delegates to the CLI settings;
+# GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL still win.
+# plan_review_tier: frontier
+# Implementation recommendation: frontier | smart. Never switches your session.
+# implementation_tier: smart
+# Optional pins: model_frontier_claude, model_frontier_openai,
+# model_smart_claude and model_smart_openai. Leave pins absent for catalog defaults;
+# empty pins are invalid. gstack-models list prints copyable set/unset commands.
+# Invalid writes are rejected and preserve the stored value.
+# Docs: docs/model-policy.md
 #
 # ─── Advanced ────────────────────────────────────────────────────────
 # codex_reviews: enabled    # Workflow outside review (Codex or Claude Code by harness).

codex/sections/challenge-mode.md

@@ -39,6 +39,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`.
 
 ```bash
 _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; }
+PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
+[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
 PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true)
 if [ -z "$PYTHON_CMD" ]; then
   echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2
@@ -54,8 +56,6 @@ if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+540)); _COD
 else _CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1; fi
 _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p')
 _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p')
-PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
-[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
 import sys, json
 turn_completed_count = 0

codex/sections/challenge-mode.md.tmpl

@@ -30,6 +30,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`.
 
 ```bash
 _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; }
+PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
+[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
 PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true)
 if [ -z "$PYTHON_CMD" ]; then
   echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2
@@ -40,8 +42,6 @@ fi
 [ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
 TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
 {{CODEX_SELECT:exec:540}}
-PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
-[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
 import sys, json
 turn_completed_count = 0

docs/model-policy.md

@@ -207,7 +207,8 @@ user-owned issues are not trusted state.
 
 A documented retain decision preserves the current default, but does not make a
 vendor mismatch `current`: the advisory issue stays open, or reopens, while that
-candidate remains. Keep the rationale outside the tool-owned report region.
+candidate remains. Record retain rationales in issue comments, which the workflow
+never edits, rather than racing a scheduled update to the issue body.
 
 Thirty-day staleness means an absence of trustworthy freshness evidence, not
 that a model was released thirty days ago. An unchanged recommendation does not

lib/claude-code.ts

@@ -2,7 +2,7 @@
 import { spawn, type ChildProcess } from 'node:child_process';
 import { resolveClaudeCommand, type ClaudeCommand } from './claude-bin';
 import { initializeWindowsReviewJob, WindowsReviewSupervisionError } from './claude-code-windows-job';
-import { ModelPolicyError, describeSelection, resolvePlanReviewModel, type ModelSelection } from './model-policy';
+import { ModelPolicyError, describeSelection, resolvePlanReviewModel, selectionRepair, type ModelSelection } from './model-policy';
 import { emitModelPolicyNotice } from './model-policy-notice';
 
 export const CLAUDE_CODE_OUTPUT_LIMIT = 32 * 1024 * 1024;
@@ -170,7 +170,18 @@ export async function runClaudeCode(options: ClaudeCodeOptions): Promise<ClaudeC
     role: selection.role, tier: selection.tier, status: selection.status,
     requested_model: selection.requestedModel, source: selection.source.label,
   };
-  const withSelection = (result: ClaudeCodeResult): ClaudeCodeResult => (summary ? { ...result, selection: summary } : result);
+  const withSelection = (result: ClaudeCodeResult): ClaudeCodeResult => {
+    if (!summary || !selection) return result;
+    const diagnostic = `${result.error?.message ?? ''}\n${result.stderr ?? ''}`;
+    const rejectedModel = /\b(?:model_not_found|model_not_supported|invalid_model)\b|\b(?:unknown|unsupported|invalid|unavailable) model\b|\bmodel\b[^\n]{0,1024}\b(?:not found|not exist|not available|not supported|unavailable)\b/i.test(diagnostic)
+      || (/\bmodel\b/i.test(diagnostic) && /\bnot_found_error\b/i.test(diagnostic));
+    if (result.status !== 'completed' && result.error && ['exit', 'provider-error'].includes(result.error.code) && rejectedModel) {
+      return { ...result, selection: summary, error: { ...result.error,
+        message: `${result.error.message} Selection: ${describeSelection(selection)}. Repair: ${selectionRepair(selection).join('; ')}. No outside review completed; model existence and account access remain provider-controlled. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#model-policy-selection`,
+      } };
+    }
+    return { ...result, selection: summary };
+  };
 
   let child: ChildProcess;
   try {

test/model-policy-cli.test.ts

@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { modelCatalogSha256 } from '../lib/model-catalog';
-import { validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy';
+import { parseModelPolicyConfig, validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy';
 import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration';
 
 const ROOT = path.resolve(import.meta.dir, '..');
@@ -230,6 +230,18 @@ describe('gstack-models inspector', () => {
 });
 
 describe('gstack-config model policy keys', () => {
+  test('the annotated header has valid uncommentable tier settings and no empty pin examples', () => {
+    const { env, state } = fixture();
+    expect(config(['set', 'telemetry', 'off'], env).code).toBe(0);
+    const text = fs.readFileSync(path.join(state, 'config.yaml'), 'utf8');
+    const tiers = text.split('\n').filter(line => /^# (plan_review_tier|implementation_tier):/.test(line)).map(line => line.slice(2));
+    expect(tiers).toHaveLength(2);
+    const parsed = parseModelPolicyConfig(tiers.join('\n'), '/example/config.yaml');
+    expect(parsed.planReviewTier.value).toBe('frontier');
+    expect(parsed.implementationTier.value).toBe('smart');
+    expect(text).not.toMatch(/^# model_(frontier|smart)_(claude|openai):\s*(?:#.*)?$/m);
+  });
+
   test('Anthropic pins preserve the 512-character contract with a POSIX-bounded regex engine', () => {
     const { root, env, state } = fixture();
     const bin = path.join(root, 'portable-bin');

test/model-role-invocation.test.ts

@@ -44,6 +44,7 @@ if (probe) {
 }
 if (!claude || args.includes('-')) await Bun.stdin.text();
 const response = process.env.FAKE_RESPONSE || 'Medium: the plan couples two rollouts.\\nRecommendation: split the migration because the plan couples two rollouts.';
+if (claude && process.env.FAKE_PROVIDER_ERROR) { console.log(JSON.stringify({ is_error: true, result: process.env.FAKE_PROVIDER_ERROR })); process.exit(Number(process.env.FAKE_ERROR_EXIT ?? '1')); }
 if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } }));
 else writeFileSync(args[args.indexOf('-o') + 1], response);
 `);
@@ -425,6 +426,32 @@ describe('one deadline bounds Codex role readiness and dispatch (native F3)', ()
 describe('Claude plan-review invocation emits one selected model or none', () => {
   const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!);
 
+  test('provider model rejections identify the winning selection and its actionable repair without substitution', () => {
+    const rejection = 'API Error: 404 {"error":{"type":"not_found_error","message":"model: requested-model"}}';
+    for (const [config, extra, repair] of [
+      ['', {}, 'gstack-config set plan_review_tier smart'],
+      ['model_frontier_claude: claude-pin\n', {}, 'gstack-config unset model_frontier_claude'],
+      ['', { GSTACK_CLAUDE_MODEL: 'claude-env' }, 'unset GSTACK_CLAUDE_MODEL'],
+      ['plan_review_tier: host\n', {}, 'choose the model in Claude Code settings'],
+    ] as const) {
+      for (const exit of ['0', '1']) {
+        const r = run('codex', home(config), { ...extra, FAKE_PROVIDER_ERROR: rejection, FAKE_ERROR_EXIT: exit });
+        expect(r.status).not.toBe(0);
+        expect(r.calls).toHaveLength(1);
+        const failed = result(r);
+        expect(failed.error.message).toContain(failed.selection.source);
+        expect(failed.error.message).toContain(repair);
+        expect(failed.error.message).toContain('#model-policy-selection');
+        expect(failed.error.message).toContain('No outside review completed');
+        if ('GSTACK_CLAUDE_MODEL' in extra) expect(failed.error.message).not.toContain('gstack-config set');
+      }
+    }
+    for (const message of ['authentication_error: invalid API key for model', 'API Error: 429 insufficient_quota for model', 'API Error: 503 server_error']) {
+      const r = run('codex', home(), { FAKE_PROVIDER_ERROR: message });
+      expect(result(r).error.message).not.toContain('gstack-config set plan_review_tier');
+    }
+  });
+
   test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => {
     const h = home();
     const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' });
@@ -537,6 +564,20 @@ describe('manual /codex entry: explicit role only, paid probe only for the dispa
     expect(role.stderr).not.toContain('NOTICE: gstack plan reviews');
   });
 
+  test('a missing manual challenge prompt stops before the paid readiness probe or notice', () => {
+    const h = home();
+    const template = fs.readFileSync(path.join(ROOT, 'codex/sections/challenge-mode.md.tmpl'), 'utf8');
+    const body = template.match(/```bash\n([\s\S]*?)\n```/)![1]!;
+    const prefix = body.slice(0, body.indexOf('"$_CODEX_PROBE" run-with-timeout'))
+      .replace('{{CODEX_SELECT:exec:540}}', RESOLVERS.CODEX_SELECT(ctxFor('codex', 'claude', HOST_PATHS.claude), ['exec', '540']))
+      .replaceAll('<prompt-file-name>', 'missing-plan-prompt');
+    const r = shell(h, `TMP_ROOT='${h.dir}'\n${withRole(prefix, 'plan-review')}`);
+    expect(r.status).toBe(1);
+    expect(r.stderr).toContain('prompt was never written');
+    expect(r.calls).toEqual([]);
+    expect(fs.existsSync(marker(h.state))).toBe(false);
+  });
+
   for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) {
     const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8');
     const selections = [...template.matchAll(/\{\{CODEX_SELECT:(exec|review):(\d+)\}\}/g)];