diff --git a/bin/gstack-config b/bin/gstack-config index 5a7ca4d94..84c46c57f 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -152,18 +152,16 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # returns to the generic overlay. # # ─── Model policy (gstack-models shows the effective choices) ─────── -# plan_review_tier: frontier # frontier | smart | host — model tier for the -# # opposing-provider plan review. host = the -# # Codex / Claude Code CLI settings choose. -# # GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL win. -# implementation_tier: smart # frontier | smart — tier gstack recommends for -# # implementing an approved plan (never switches -# # your session). -# model_frontier_claude: # per-tier model IDs; unset = the shipped catalog -# model_frontier_openai: # recommendation (gstack-models list). Invalid -# model_smart_claude: # values are REJECTED and the stored value kept; -# model_smart_openai: # unset restores the shipped recommendation. -# # Docs: docs/model-policy.md +# Plan review tier: frontier | smart | host. host delegates to the CLI settings; +# GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL still win. +# plan_review_tier: frontier +# Implementation recommendation: frontier | smart. Never switches your session. +# implementation_tier: smart +# Optional pins: model_frontier_claude, model_frontier_openai, +# model_smart_claude and model_smart_openai. Leave pins absent for catalog defaults; +# empty pins are invalid. gstack-models list prints copyable set/unset commands. +# Invalid writes are rejected and preserve the stored value. +# Docs: docs/model-policy.md # # ─── Advanced ──────────────────────────────────────────────────────── # codex_reviews: enabled # Workflow outside review (Codex or Claude Code by harness). diff --git a/codex/sections/challenge-mode.md b/codex/sections/challenge-mode.md index 2e2cf8648..1e0294de1 100644 --- a/codex/sections/challenge-mode.md +++ b/codex/sections/challenge-mode.md @@ -39,6 +39,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`. ```bash _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; } +PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" +[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true) if [ -z "$PYTHON_CMD" ]; then echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2 @@ -54,8 +56,6 @@ if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+540)); _COD else _CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1; fi _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" -[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c " import sys, json turn_completed_count = 0 diff --git a/codex/sections/challenge-mode.md.tmpl b/codex/sections/challenge-mode.md.tmpl index 278071ef6..49c3f1db5 100644 --- a/codex/sections/challenge-mode.md.tmpl +++ b/codex/sections/challenge-mode.md.tmpl @@ -30,6 +30,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`. ```bash _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; } +PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" +[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true) if [ -z "$PYTHON_CMD" ]; then echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2 @@ -40,8 +42,6 @@ fi [ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } {{CODEX_SELECT:exec:540}} -PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" -[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c " import sys, json turn_completed_count = 0 diff --git a/docs/model-policy.md b/docs/model-policy.md index 274d46adf..6ca945a28 100644 --- a/docs/model-policy.md +++ b/docs/model-policy.md @@ -207,7 +207,8 @@ user-owned issues are not trusted state. A documented retain decision preserves the current default, but does not make a vendor mismatch `current`: the advisory issue stays open, or reopens, while that -candidate remains. Keep the rationale outside the tool-owned report region. +candidate remains. Record retain rationales in issue comments, which the workflow +never edits, rather than racing a scheduled update to the issue body. Thirty-day staleness means an absence of trustworthy freshness evidence, not that a model was released thirty days ago. An unchanged recommendation does not diff --git a/lib/claude-code.ts b/lib/claude-code.ts index dec84592f..28f39ff6f 100644 --- a/lib/claude-code.ts +++ b/lib/claude-code.ts @@ -2,7 +2,7 @@ import { spawn, type ChildProcess } from 'node:child_process'; import { resolveClaudeCommand, type ClaudeCommand } from './claude-bin'; import { initializeWindowsReviewJob, WindowsReviewSupervisionError } from './claude-code-windows-job'; -import { ModelPolicyError, describeSelection, resolvePlanReviewModel, type ModelSelection } from './model-policy'; +import { ModelPolicyError, describeSelection, resolvePlanReviewModel, selectionRepair, type ModelSelection } from './model-policy'; import { emitModelPolicyNotice } from './model-policy-notice'; export const CLAUDE_CODE_OUTPUT_LIMIT = 32 * 1024 * 1024; @@ -170,7 +170,18 @@ export async function runClaudeCode(options: ClaudeCodeOptions): Promise (summary ? { ...result, selection: summary } : result); + const withSelection = (result: ClaudeCodeResult): ClaudeCodeResult => { + if (!summary || !selection) return result; + const diagnostic = `${result.error?.message ?? ''}\n${result.stderr ?? ''}`; + const rejectedModel = /\b(?:model_not_found|model_not_supported|invalid_model)\b|\b(?:unknown|unsupported|invalid|unavailable) model\b|\bmodel\b[^\n]{0,1024}\b(?:not found|not exist|not available|not supported|unavailable)\b/i.test(diagnostic) + || (/\bmodel\b/i.test(diagnostic) && /\bnot_found_error\b/i.test(diagnostic)); + if (result.status !== 'completed' && result.error && ['exit', 'provider-error'].includes(result.error.code) && rejectedModel) { + return { ...result, selection: summary, error: { ...result.error, + message: `${result.error.message} Selection: ${describeSelection(selection)}. Repair: ${selectionRepair(selection).join('; ')}. No outside review completed; model existence and account access remain provider-controlled. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#model-policy-selection`, + } }; + } + return { ...result, selection: summary }; + }; let child: ChildProcess; try { diff --git a/test/model-policy-cli.test.ts b/test/model-policy-cli.test.ts index 3b06b5a0d..db9996244 100644 --- a/test/model-policy-cli.test.ts +++ b/test/model-policy-cli.test.ts @@ -4,7 +4,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { modelCatalogSha256 } from '../lib/model-catalog'; -import { validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy'; +import { parseModelPolicyConfig, validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy'; import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -230,6 +230,18 @@ describe('gstack-models inspector', () => { }); describe('gstack-config model policy keys', () => { + test('the annotated header has valid uncommentable tier settings and no empty pin examples', () => { + const { env, state } = fixture(); + expect(config(['set', 'telemetry', 'off'], env).code).toBe(0); + const text = fs.readFileSync(path.join(state, 'config.yaml'), 'utf8'); + const tiers = text.split('\n').filter(line => /^# (plan_review_tier|implementation_tier):/.test(line)).map(line => line.slice(2)); + expect(tiers).toHaveLength(2); + const parsed = parseModelPolicyConfig(tiers.join('\n'), '/example/config.yaml'); + expect(parsed.planReviewTier.value).toBe('frontier'); + expect(parsed.implementationTier.value).toBe('smart'); + expect(text).not.toMatch(/^# model_(frontier|smart)_(claude|openai):\s*(?:#.*)?$/m); + }); + test('Anthropic pins preserve the 512-character contract with a POSIX-bounded regex engine', () => { const { root, env, state } = fixture(); const bin = path.join(root, 'portable-bin'); diff --git a/test/model-role-invocation.test.ts b/test/model-role-invocation.test.ts index 8220e61fd..1af5b2996 100644 --- a/test/model-role-invocation.test.ts +++ b/test/model-role-invocation.test.ts @@ -44,6 +44,7 @@ if (probe) { } if (!claude || args.includes('-')) await Bun.stdin.text(); const response = process.env.FAKE_RESPONSE || 'Medium: the plan couples two rollouts.\\nRecommendation: split the migration because the plan couples two rollouts.'; +if (claude && process.env.FAKE_PROVIDER_ERROR) { console.log(JSON.stringify({ is_error: true, result: process.env.FAKE_PROVIDER_ERROR })); process.exit(Number(process.env.FAKE_ERROR_EXIT ?? '1')); } if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } })); else writeFileSync(args[args.indexOf('-o') + 1], response); `); @@ -425,6 +426,32 @@ describe('one deadline bounds Codex role readiness and dispatch (native F3)', () describe('Claude plan-review invocation emits one selected model or none', () => { const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!); + test('provider model rejections identify the winning selection and its actionable repair without substitution', () => { + const rejection = 'API Error: 404 {"error":{"type":"not_found_error","message":"model: requested-model"}}'; + for (const [config, extra, repair] of [ + ['', {}, 'gstack-config set plan_review_tier smart'], + ['model_frontier_claude: claude-pin\n', {}, 'gstack-config unset model_frontier_claude'], + ['', { GSTACK_CLAUDE_MODEL: 'claude-env' }, 'unset GSTACK_CLAUDE_MODEL'], + ['plan_review_tier: host\n', {}, 'choose the model in Claude Code settings'], + ] as const) { + for (const exit of ['0', '1']) { + const r = run('codex', home(config), { ...extra, FAKE_PROVIDER_ERROR: rejection, FAKE_ERROR_EXIT: exit }); + expect(r.status).not.toBe(0); + expect(r.calls).toHaveLength(1); + const failed = result(r); + expect(failed.error.message).toContain(failed.selection.source); + expect(failed.error.message).toContain(repair); + expect(failed.error.message).toContain('#model-policy-selection'); + expect(failed.error.message).toContain('No outside review completed'); + if ('GSTACK_CLAUDE_MODEL' in extra) expect(failed.error.message).not.toContain('gstack-config set'); + } + } + for (const message of ['authentication_error: invalid API key for model', 'API Error: 429 insufficient_quota for model', 'API Error: 503 server_error']) { + const r = run('codex', home(), { FAKE_PROVIDER_ERROR: message }); + expect(result(r).error.message).not.toContain('gstack-config set plan_review_tier'); + } + }); + test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => { const h = home(); const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' }); @@ -537,6 +564,20 @@ describe('manual /codex entry: explicit role only, paid probe only for the dispa expect(role.stderr).not.toContain('NOTICE: gstack plan reviews'); }); + test('a missing manual challenge prompt stops before the paid readiness probe or notice', () => { + const h = home(); + const template = fs.readFileSync(path.join(ROOT, 'codex/sections/challenge-mode.md.tmpl'), 'utf8'); + const body = template.match(/```bash\n([\s\S]*?)\n```/)![1]!; + const prefix = body.slice(0, body.indexOf('"$_CODEX_PROBE" run-with-timeout')) + .replace('{{CODEX_SELECT:exec:540}}', RESOLVERS.CODEX_SELECT(ctxFor('codex', 'claude', HOST_PATHS.claude), ['exec', '540'])) + .replaceAll('', 'missing-plan-prompt'); + const r = shell(h, `TMP_ROOT='${h.dir}'\n${withRole(prefix, 'plan-review')}`); + expect(r.status).toBe(1); + expect(r.stderr).toContain('prompt was never written'); + expect(r.calls).toEqual([]); + expect(fs.existsSync(marker(h.state))).toBe(false); + }); + for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) { const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8'); const selections = [...template.matchAll(/\{\{CODEX_SELECT:(exec|review):(\d+)\}\}/g)];