feat: add configurable model tiers and advisory freshness

This commit is contained in:
garrytan committed 2026-10-07 23:41:24 +00:00
1 parent db745675bd
commit 5ffc050659
78 files changed
+5106 -325

No files matched your search

@@ -0,0 +1,49 @@
name: Model-policy freshness
on:
schedule:
- cron: '0 11 * * 1'
workflow_dispatch:
permissions:
contents: read
concurrency:
group: model-policy-freshness
cancel-in-progress: false
jobs:
advisory:
runs-on: ubuntu-24.04
timeout-minutes: 8
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.2
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Check and publish main advisory
if: github.ref == 'refs/heads/main'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json
run: bun scripts/model-policy-freshness.ts --publish
- name: Check non-main without publication
if: github.ref != 'refs/heads/main'
env:
MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json
run: bun scripts/model-policy-freshness.ts
- name: Preserve evidence report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: model-policy-freshness-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/model-policy-freshness.json
if-no-files-found: error
retention-days: 90
+3 -2
View File
@@ -173,8 +173,9 @@ defaults are `claude-fable-5-1`, resolved through `lib/eval-model.ts`:
Warmup stays on `claude-haiku-4-5`; distill stays on
`claude-haiku-4-5-20251001`. Explicit test and historical benchmark model
selections still win. Known frontier defaults are maintained in releases;
there is no automatic model discovery. Paid-run costs shown below are
selections still win. These eval defaults stay pinned independently from the
[runtime model tiers](docs/model-policy.md); the tier freshness workflow does
not discover or change evaluation models automatically. Paid-run costs shown below are
historical estimates from before this default change, not measurements of
the new defaults.
+13 -3
View File
@@ -165,8 +165,18 @@ override applies to that run only; set `model` in your Codex `config.toml` to
make it stick across upgrades. After changing your Codex model, rerun
`./setup --host codex` to regenerate the skills.
**Which Codex model gstack uses.** For every Codex call (outside voices,
`/codex`, review and ship adversarial passes), gstack picks the model in this
**Plan-review and implementation tiers.** Plan workflows use an independent
`frontier` reviewer: Fable 5.1 from Codex, or GPT-6 Astra from Claude Code.
Implementation handoffs recommend the `smart` tier: Opus 5.5 or GPT-6.1 Sol.
Choose a different tier, pin either provider's model, or use `host` mode to
preserve native model settings. The read-only `gstack-models` command explains
the effective choices; changing these settings does not require regenerating
skills or switch your running session. A weekly maintenance check flags changed
official recommendations and retirement notices without automatically changing
models. See [model policy: setup, overrides and freshness](docs/model-policy.md).
**Other Codex calls keep their existing selection.** Without the explicit
`plan-review` role (including ordinary review and ship adversarial passes), gstack picks the model in this
order: a model you name for that request, then `GSTACK_CODEX_MODEL`, then
`model` in your Codex `config.toml` (for native `codex review`, `review_model`
first; a custom `CODEX_HOME` is honored), and only then gstack's default,
@@ -177,7 +187,7 @@ repair message and reports the outside review as unavailable. It never silently
switches to its default. Nested Codex reviews also run with installed skills
hidden (`-c skills.include_instructions=false`), so a review cannot turn into a
whole nested skill run. Runtime model selection is separate from the setup-time
behavioral profile above. `/claude-code` (`gstack-claude-code`
behavioral profile above. Without the plan-review role, `/claude-code` (`gstack-claude-code`
on Codex) preserves Claude's configured model. Set `GSTACK_CLAUDE_MODEL=<model>`
or name a model in your request to override it for the invocation, including
resumed consultations. See [eval defaults and overrides](CONTRIBUTING.md#testing--evals)
+18 -45
View File
@@ -754,56 +754,27 @@ Review skills will load at each phase entry. Starting full review pipeline with
```bash
# Codex preflight: one block (functions sourced here don't persist to later blocks).
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
_gstack_helper_error=""
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
if [ "$_CODEX_CFG" = "disabled" ]; then
_CODEX_MODE="disabled"
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
_CODEX_MODE="under_codex"
elif ! command -v codex >/dev/null 2>&1; then
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
elif [ -n "$_gstack_helper_error" ]; then
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
else
# The free sandbox check runs before the paid model probe. Probe code 2 means
# the CLI cannot execute at all, a different fix from an unusable model.
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
if [ "$_CODEX_MP" -eq 3 ]; then
_CODEX_MODE="sandbox_unavailable"
elif [ "$_CODEX_MP" -eq 2 ]; then
_CODEX_MODE="broken_install"
elif [ "$_CODEX_MP" -eq 4 ]; then
_CODEX_MODE="quota_exhausted"
elif [ "$_CODEX_MP" -ne 0 ]; then
_CODEX_MODE="model_unusable"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
_CODEX_MODE="unverified"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
_CODEX_MODE="unverified (rate_limited)"
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
if [ "$_OUTSIDE_CFG" = disabled ]; then
echo 'CODEX_MODE: disabled'
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
else
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
fi
exit 78
fi
); then
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
else
echo 'CODEX_MODE: under_current_harness'
fi
echo "CODEX_MODE: $_CODEX_MODE"
```
Branch on the echoed `CODEX_MODE`:
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only."
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider.
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Keep the required Claude adversarial pass; do not dispatch a duplicate. The ~10s round trip is cached for 1h.
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Keep the required Claude adversarial pass; do not dispatch a duplicate.
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled skips only the outside CLI; retain the native pass. Non-ready means missing outside coverage. Keep the required native pass without duplicating it. Never substitute another external provider.
Disabled/unavailable retains applicable native passes. Recheck each outside dispatch.
Record provider and completed/unavailable/disabled/skipped per phase; CEO covers
@@ -990,4 +961,6 @@ Retain the historical review-log skill ID; add `"host":"claude","outside_provide
Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status,
native completion, findings, and partial coverage. Replace N with actual counts.
**Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
Suggest next step: `/ship` when ready to create the PR.
+2
View File
@@ -464,4 +464,6 @@ disabled or skipped. Never carry success across phases/runs; preserve modelUsage
Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status,
native completion, findings, and partial coverage. Replace N with actual counts.
{{IMPLEMENTATION_MODEL_HANDOFF}}
Suggest next step: `/ship` when ready to create the PR.
+1 -2
View File
@@ -111,8 +111,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+1 -2
View File
@@ -70,8 +70,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+1 -2
View File
@@ -74,8 +74,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+1 -2
View File
@@ -63,8 +63,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+234 -36
View File
@@ -10,6 +10,8 @@
# _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
# _gstack_codex_model_probe — round-trip probe of the selected model (#2477);
# exit 4 = the account's Codex usage limit
# _gstack_codex_lock_acquire — bounded mkdir lock with abandoned-owner recovery
# _gstack_codex_role_ready — plan-review selection, auth, sandbox and model probe in one shell
# _gstack_codex_version_check — warn on known-bad Codex CLI versions
# _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
# _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/
@@ -114,33 +116,65 @@ _gstack_codex_select_model() {
# One selection record per invocation kind, shared by the probe and the
# dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL,
# Codex config.toml (review_model first for native review), gstack default.
# With $3 = plan-review the plan-review policy ranks the choice instead and
# its record carries role, tier, source and source-aware repair; $4 names the
# target repository whose project settings can route to a custom endpoint.
# The TOML is read by the TypeScript resolver; bash never parses it.
local _kind="${1:-exec}" _explicit="${2:-}" _line _rc
local _kind="${1:-exec}" _explicit="${2:-}" _role="${3:-}" _cwd="${4:-}" _line _rc _rest
case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac
case "$_role" in ''|plan-review) ;; *) echo "ERROR: _gstack_codex_select_model role must be plan-review" >&2; return 2 ;; esac
if ! command -v bun >/dev/null 2>&1; then
echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2
return 1
fi
if [ -n "$_explicit" ]; then
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit")
else
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind")
fi
# Function-local argv: each value stays one literal argument in bash and zsh.
set -- --runtime "$_kind"
[ -n "$_role" ] && set -- "$@" --role "$_role"
[ -n "$_explicit" ] && set -- "$@" --explicit "$_explicit"
[ -n "$_cwd" ] && set -- "$@" --cwd "$_cwd"
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" "$@")
_rc=$?
_GSTACK_CODEX_SEL="${_line%%$'\t'*}"
_GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}"
_rest="${_line#*$'\t'}"
_GSTACK_CODEX_SEL_TIER=""
_GSTACK_CODEX_SEL_REPAIR=""
if [ -n "$_role" ]; then
_GSTACK_CODEX_SEL_SRC="${_rest%%$'\t'*}"
_rest="${_rest#*$'\t'}"
_GSTACK_CODEX_SEL_TIER="${_rest%%$'\t'*}"
_GSTACK_CODEX_SEL_REPAIR="${_rest#*$'\t'}"
else
_GSTACK_CODEX_SEL_SRC="$_rest"
fi
case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac
if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then
_GSTACK_CODEX_SEL=""
_GSTACK_CODEX_SEL_SRC=""
_GSTACK_CODEX_SEL_ROLE=""
echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2
return 1
fi
_GSTACK_CODEX_SEL_KIND="$_kind"
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
_GSTACK_CODEX_SEL_ROLE="$_role"
_GSTACK_CODEX_SEL_CWD="$_cwd"
if [ -n "$_role" ]; then
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; role: $_role, tier: $_GSTACK_CODEX_SEL_TIER; source: $_GSTACK_CODEX_SEL_SRC)" >&2
else
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
fi
_gstack_codex_sandbox_mode
}
_gstack_codex_role_ready() {
# $1 exec|review, $2 target repository. Resolves the plan-review model once and
# checks auth, the sandbox and that model; the caller dispatches the same record.
_gstack_codex_select_model "$1" '' plan-review "${2:-}" || return 1
_gstack_codex_auth_probe || return 1
_gstack_codex_sandbox_preflight >/dev/null || return 1
_gstack_codex_version_check
_gstack_codex_model_probe "$1"
}
# --- Sandbox (B1) -----------------------------------------------------------
_gstack_codex_sandbox_mode() {
@@ -246,7 +280,7 @@ _gstack_codex_model_probe() {
# auth just measures the auth failure again.
local _kind="${1:-exec}"
if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then
if ! _gstack_codex_select_model "$_kind"; then
if ! _gstack_codex_select_model "$_kind" "" "${_GSTACK_CODEX_SEL_ROLE:-}" "${_GSTACK_CODEX_SEL_CWD:-}"; then
echo "MODEL_UNUSABLE"
echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
return 1
@@ -286,36 +320,42 @@ _gstack_codex_model_probe() {
local _now
_now=$(date +%s 2>/dev/null || echo 0)
local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md"
if [ -n "$_cache" ] && [ -f "$_cache" ]; then
local _c_line _c_status _c_ts _c_sig
_c_line=$(head -1 "$_cache" 2>/dev/null)
_c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1)
_c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2)
_c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3)
case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac
if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
echo "MODEL_OK (cached)"
return 0
# Up to 16 signatures share one cache file. Same-signature misses serialize
# on a bounded lock and recheck the cache, so concurrent callers pay for one
# round trip. A lock timeout or unreadable cache is unverified readiness,
# never a duplicate paid probe; the review's own result still decides.
local _lock="" _rc
if [ -n "$_cache" ]; then
_gstack_codex_probe_cache_hit; _rc=$?
[ "$_rc" -lt 10 ] && return "$_rc"
if [ "$_rc" -eq 11 ]; then _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; fi
_lock="$_gstack_home/.codex-model-probe.locks/$_sig"
if ! mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null; then
_gstack_codex_probe_unverified "cannot create the probe lock under $_gstack_home"; return 0
fi
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry"
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
echo "MODEL_QUOTA_EXHAUSTED (cached)"
sed -n '2,4p' "$_cache" 2>/dev/null
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
return 4
fi
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
echo "MODEL_UNUSABLE (cached)"
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
return 1
_gstack_codex_lock_acquire "$_lock" "${_GSTACK_CODEX_LOCK_WAIT:-45}" "${_GSTACK_CODEX_LOCK_STALE:-90}"; _rc=$?
if [ "$_rc" -eq 1 ]; then _gstack_codex_probe_unverified "another probe of this model still holds its lock"; return 0; fi
if [ "$_rc" -ne 0 ]; then _gstack_codex_probe_unverified "cannot record ownership of the probe lock under $_gstack_home"; return 0; fi
_gstack_codex_probe_cache_hit; _rc=$?
if [ "$_rc" -lt 10 ] || [ "$_rc" -eq 11 ]; then
_gstack_codex_lock_release "$_lock"
[ "$_rc" -eq 11 ] && { _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; }
return "$_rc"
fi
fi
_gstack_codex_probe_round_trip; _rc=$?
[ -n "$_lock" ] && _gstack_codex_lock_release "$_lock"
return "$_rc"
}
_gstack_codex_probe_round_trip() {
# The paid round trip and its classification. Called only by
# _gstack_codex_model_probe; reads its locals (_model, _sig, _cache, ...).
local _out _code
_out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" </dev/null 2>&1)
_code=$?
if [ "$_code" -eq 0 ]; then
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
_gstack_codex_probe_cache_store MODEL_OK
echo "MODEL_OK"
return 0
fi
@@ -335,7 +375,7 @@ _gstack_codex_model_probe() {
fi
_q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3)
if [ -n "$_q_lines" ]; then
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_QUOTA_EXHAUSTED %s %s\n%s\n' "$_now" "$_sig" "$_q_lines" > "$_cache" 2>/dev/null; }
_gstack_codex_probe_cache_store MODEL_QUOTA_EXHAUSTED "$_q_lines"
echo "MODEL_QUOTA_EXHAUSTED"
printf '%s\n' "$_q_lines"
echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted"
@@ -357,7 +397,7 @@ _gstack_codex_model_probe() {
# base_url answers a bare 404; neither is network luck, so neither may fall
# through to the inconclusive branch below and report ready (#2843).
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
_gstack_codex_probe_cache_store MODEL_UNUSABLE
echo "MODEL_UNUSABLE"
printf '%s\n' "$_out" | grep -iE "model|404" | head -3
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
@@ -366,10 +406,14 @@ _gstack_codex_model_probe() {
_gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
echo "HINT: choose a current model ($_GSTACK_CODEX_SEL_REPAIR), or fix that provider's base_url."
else
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
fi
_gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
else
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
_gstack_codex_model_repair_hint
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
fi
return 1
@@ -400,6 +444,160 @@ _gstack_codex_model_probe() {
return 0
}
_gstack_codex_model_repair_hint() {
# Repair for the record actually selected: a plan-review record names its
# winning source (request, env, tier override, catalog or host mode).
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
echo "HINT: choose a model your account can use: $_GSTACK_CODEX_SEL_REPAIR"
else
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in ${_codex_home:-\$CODEX_HOME}/config.toml, or name one for this request."
fi
}
_gstack_codex_probe_unverified() {
_GSTACK_CODEX_PROBE_STATE="inconclusive"
echo "MODEL_PROBE_INCONCLUSIVE ($1) — CODEX_MODE: unverified; no duplicate probe was made, and the review's own result is still checked."
}
_gstack_codex_probe_cache_hit() {
# Cache file: one "STATUS EPOCH SIGNATURE" line per entry, newest last, each
# followed by its tab-prefixed Codex lines. The pre-v2 one-entry file has the
# same first line, so it migrates by TTL; its unprefixed lines are ignored.
# Returns 0/1/4 after printing a hit, 10 on a miss, 11 when unreadable.
[ -e "$_cache" ] || return 10
{ [ -f "$_cache" ] && [ -r "$_cache" ]; } || return 11
local _hit _head _c_status _c_ts _detail=""
_hit=$(awk -v sig="$_sig" '
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ { cur = ($3 == sig); if (cur) out = $1 " " $2; next }
/^\t/ { if (cur) out = out "\n" substr($0, 2); next }
{ cur = 0 }
END { if (out != "") print out }' "$_cache" 2>/dev/null) || return 11
[ -n "$_hit" ] || return 10
_head="${_hit%%$'\n'*}"
[ "$_head" != "$_hit" ] && _detail="${_hit#*$'\n'}"
_c_status="${_head%% *}"
_c_ts="${_head#* }"
case "$_c_ts" in ''|*[!0-9]*) return 10 ;; esac
[ "$_c_ts" -le "$_now" ] || return 10
if [ "$_c_status" = "MODEL_OK" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
echo "MODEL_OK (cached)"
return 0
fi
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && return 10
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ $((_now - _c_ts)) -lt 900 ]; then
echo "MODEL_QUOTA_EXHAUSTED (cached)"
[ -n "$_detail" ] && printf '%s\n' "$_detail"
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
return 4
fi
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ $((_now - _c_ts)) -lt 900 ]; then
echo "MODEL_UNUSABLE (cached)"
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
_gstack_codex_model_repair_hint
return 1
fi
return 10
}
_gstack_codex_probe_cache_store() {
# $1 status, $2 optional Codex lines. Rewrites the file under a short write
# lock and publishes it with one rename: this signature's entry replaces any
# older one, expired entries drop, and at most 16 entries remain. A failure
# leaves the previous file intact; the probe's own result still stands.
[ -n "$_cache" ] || return 0
[ -d "$_cache" ] && return 1
local _w="$_gstack_home/.codex-model-probe.locks/write" _tmp="$_cache.tmp.$$" _wrc=0
mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null || return 1
_gstack_codex_lock_acquire "$_w" 5 30 || return 1
: > "$_tmp" 2>/dev/null || _wrc=1
if [ "$_wrc" -eq 0 ] && [ -f "$_cache" ]; then
awk -v now="$_now" -v sig="$_sig" -v keep=15 '
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ {
ttl = ($1 == "MODEL_OK") ? 3600 : 900
cur = ($3 != sig && $2 <= now && now - $2 < ttl)
if (cur) entry[++n] = $0
next
}
/^\t/ { if (cur) entry[n] = entry[n] "\n" $0; next }
{ cur = 0 }
END { for (i = (n > keep ? n - keep + 1 : 1); i <= n; i++) print entry[i] }' "$_cache" > "$_tmp" 2>/dev/null || _wrc=1
fi
if [ "$_wrc" -eq 0 ]; then
{
printf '%s %s %s\n' "$1" "$_now" "$_sig"
[ -z "${2:-}" ] || printf '%s\n' "$2" | awk '{ print "\t" $0 }'
} >> "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_cache" 2>/dev/null || _wrc=1
fi
rm -f "$_tmp" 2>/dev/null
_gstack_codex_lock_release "$_w"
return "$_wrc"
}
# --- Bounded lock -----------------------------------------------------------
_gstack_codex_lock_acquire() {
# $1 lock directory, $2 seconds to wait, $3 seconds after which even a live
# owner is abandoned. mkdir is atomic on every platform bash runs on. A dead
# owner on this host is recovered at once; a missing owner file is the
# creation window and becomes stale after 5 seconds. 0 = held, 1 = timed
# out, 2 = the lock or its owner record cannot be created.
local _dir="$1" _tries=$(( ${2:-45} * 10 )) _stale="${3:-90}" _host _owner _pid _ohost _ts _now _i=0 _dead
_host=$(uname -n 2>/dev/null || echo unknown)
while :; do
if mkdir "$_dir" 2>/dev/null; then
printf '%s %s %s\n' "$$" "$_host" "$(date +%s 2>/dev/null || echo 0)" > "$_dir/owner" 2>/dev/null && return 0
rm -f "$_dir/owner" 2>/dev/null
rmdir "$_dir" 2>/dev/null
return 2
fi
[ -d "$_dir" ] || return 2
_now=$(date +%s 2>/dev/null || echo 0)
_owner=$(cat "$_dir/owner" 2>/dev/null)
_dead=0
if [ -n "$_owner" ]; then
read -r _pid _ohost _ts <<< "$_owner"
case "$_pid" in ''|*[!0-9]*) _pid=0 ;; esac
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
if [ "$_ohost" = "$_host" ] && ! kill -0 "$_pid" 2>/dev/null; then _dead=1; fi
[ $((_now - _ts)) -ge "$_stale" ] && _dead=1
else
_ts=$(stat -c %Y "$_dir" 2>/dev/null || stat -f %m "$_dir" 2>/dev/null || echo 0)
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
[ $((_now - _ts)) -ge 5 ] && _dead=1
fi
[ "$_dead" -eq 1 ] && _gstack_codex_lock_break "$_dir" "$_owner"
[ "$_i" -ge "$_tries" ] && return 1
_i=$((_i + 1))
sleep 0.1
done
}
_gstack_codex_lock_break() {
# Breakers serialize on $1.break and remove the lock only if it still has the
# owner they judged abandoned, so a lock re-taken meanwhile survives.
local _dir="$1" _seen="$2" _bt _now
if mkdir "$_dir.break" 2>/dev/null; then
[ "$(cat "$_dir/owner" 2>/dev/null)" = "$_seen" ] && rm -rf "$_dir" 2>/dev/null
rmdir "$_dir.break" 2>/dev/null
return 0
fi
_now=$(date +%s 2>/dev/null || echo 0)
_bt=$(stat -c %Y "$_dir.break" 2>/dev/null || stat -f %m "$_dir.break" 2>/dev/null || echo "$_now")
case "$_bt" in ''|*[!0-9]*) _bt="$_now" ;; esac
[ $((_now - _bt)) -ge 10 ] && rmdir "$_dir.break" 2>/dev/null
return 0
}
_gstack_codex_lock_release() {
# Remove only a lock this shell still owns.
local _host
_host=$(uname -n 2>/dev/null || echo unknown)
case "$(cat "$1/owner" 2>/dev/null)" in
"$$ $_host "*) rm -rf "$1" 2>/dev/null ;;
esac
return 0
}
# --- Version check ----------------------------------------------------------
_gstack_codex_version_check() {
+44 -2
View File
@@ -151,6 +151,20 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne
# # ID and re-renders; ./setup --claude-model claude
# # returns to the generic overlay.
#
# ─── Model policy (gstack-models shows the effective choices) ───────
# plan_review_tier: frontier # frontier | smart | host — model tier for the
# # opposing-provider plan review. host = the
# # Codex / Claude Code CLI settings choose.
# # GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL win.
# implementation_tier: smart # frontier | smart — tier gstack recommends for
# # implementing an approved plan (never switches
# # your session).
# model_frontier_claude: # per-tier model IDs; empty = the shipped catalog
# model_frontier_openai: # recommendation (gstack-models list). Invalid
# model_smart_claude: # values are REJECTED and the stored value kept;
# model_smart_openai: # unset restores the shipped recommendation.
# # Docs: docs/model-policy.md
#
# ─── Advanced ────────────────────────────────────────────────────────
# codex_reviews: enabled # Workflow outside review (Codex or Claude Code by harness).
# # enabled: /review, /ship, /document-release, plan reviews,
@@ -278,6 +292,9 @@ lookup_default() {
ship_measure_sweep_cases) echo "5" ;;
ship_measure_sweep_budget_usd) echo "150" ;;
ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in)
plan_review_tier) echo "frontier" ;; # frontier | smart | host — lib/model-policy.ts reads it strictly
implementation_tier) echo "smart" ;; # frontier | smart
model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai) echo "" ;; # empty = shipped catalog ID (gstack-models list)
# Unknown key: exit non-zero instead of printing "". The fallback pattern
# the preambles use,
# VAR=$(gstack-config get <key> 2>/dev/null || echo "<default>")
@@ -678,6 +695,27 @@ case "${1:-}" in
echo "Error: codex_reviews '$VALUE' not recognized. Valid values: enabled, disabled. Existing value left unchanged." >&2
exit 1
fi
case "$KEY" in
plan_review_tier|implementation_tier|model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai)
_mp_doc="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#model-policy-config"
VALUE="${VALUE#"${VALUE%%[![:space:]]*}"}"
VALUE="${VALUE%"${VALUE##*[![:space:]]}"}"
_mp_valid=1
case "$KEY" in
plan_review_tier) case "$VALUE" in frontier|smart|host) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart, host." ;; esac ;;
implementation_tier) case "$VALUE" in frontier|smart) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart." ;; esac ;;
*_claude) printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[][A-Za-z0-9._:/@+-]{1,512}$' || { _mp_valid=""; _mp_rule="An Anthropic model ID is 1-512 characters from A-Z a-z 0-9 . _ : / @ + [ ] -."; } ;;
*_openai) printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[A-Za-z0-9._:/-]{1,100}$' || { _mp_valid=""; _mp_rule="An OpenAI model ID is 1-100 characters from A-Z a-z 0-9 . _ : / -."; } ;;
esac
if [ -z "$VALUE" ]; then
echo "Error: $KEY value is empty. Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2
exit 1
fi
if [ -z "$_mp_valid" ]; then
echo "Error: $KEY '$VALUE' not recognized. $_mp_rule Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2
exit 1
fi ;;
esac
# The measure-loop controls bound paid spend: reject a typo instead of
# coercing it, so a bad value never changes how much /ship may spend.
case "$KEY" in
@@ -882,7 +920,9 @@ case "${1:-}" in
timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \
browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \
ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \
plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \
model_smart_claude model_smart_openai; do
gstack_config_select "$KEY"
VALUE="$_gstack_cfg_value"
SOURCE="default"
@@ -905,7 +945,9 @@ case "${1:-}" in
timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \
browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \
ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \
plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \
model_smart_claude model_smart_openai; do
printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")"
done
;;
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bun
import { modelsMain } from '../lib/model-policy-cli.ts';
const result = modelsMain(process.argv.slice(2));
if (result.stdout) process.stdout.write(result.stdout);
if (result.stderr) process.stderr.write(result.stderr);
process.exit(result.code);
+3
View File
@@ -53,6 +53,9 @@ Set `GSTACK_CLAUDE_MODEL=<model>` for an explicit override, including resumed
consultations. If the user names a model, pass that value through this environment
variable for every runner call. Without an override, retain Claude's configured
model; harness routing never chooses a model family.
Only for an explicit `--role plan-review` (e.g. `/claude-code challenge --role plan-review`),
append `--role plan-review` to every runner call; the runner prints `CLAUDE_MODEL:` and
passes that one policy model (none in host mode).
Do not infer authentication state from credential files or environment variables.
Run the actual runner invocation in the host's normal execution context. On a
+5 -1
View File
@@ -473,9 +473,13 @@ then `gpt-6-astra`. Each call prints `CODEX_MODEL: <model> (<kind>; source: ...)
For a named model, pass it as the second argument of every `_gstack_codex_select_model`
call, and run `_gstack_codex_select_model exec '<model>'` before the probe below. An
invalid or unavailable choice stops with a repair message, never the default.
Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`),
set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then
probes and dispatches the policy model.
```bash
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_ROLE=''
source ~/.claude/skills/gstack/bin/gstack-codex-probe || { echo "HELPER_UNAVAILABLE"; exit 1; }
# GSTACK_ACTIVE_HOST names the harness, never the model.
@@ -491,7 +495,7 @@ fi
if ! _gstack_codex_auth_probe >/dev/null; then
_gstack_codex_log_event "codex_auth_failed"
echo "AUTH_FAILED"
elif _gstack_codex_sandbox_preflight; then # free; Linux only
elif _gstack_codex_sandbox_preflight && [ -z "$_CODEX_ROLE" ]; then # free; Linux only
_gstack_codex_model_probe # ~10s round trip on first run, cached 1h
fi
_gstack_codex_version_check # warns if known-bad, non-blocking
+5 -1
View File
@@ -74,16 +74,20 @@ then `gpt-6-astra`. Each call prints `CODEX_MODEL: <model> (<kind>; source: ...)
For a named model, pass it as the second argument of every `_gstack_codex_select_model`
call, and run `_gstack_codex_select_model exec '<model>'` before the probe below. An
invalid or unavailable choice stops with a repair message, never the default.
Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`),
set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then
probes and dispatches the policy model.
```bash
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_ROLE=''
source ~/.claude/skills/gstack/bin/gstack-codex-probe || { echo "HELPER_UNAVAILABLE"; exit 1; }
{{OUTSIDE_SELF_GUARD:codex}}
if ! _gstack_codex_auth_probe >/dev/null; then
_gstack_codex_log_event "codex_auth_failed"
echo "AUTH_FAILED"
elif _gstack_codex_sandbox_preflight; then # free; Linux only
elif _gstack_codex_sandbox_preflight && [ -z "$_CODEX_ROLE" ]; then # free; Linux only
_gstack_codex_model_probe # ~10s round trip on first run, cached 1h
fi
_gstack_codex_version_check # warns if known-bad, non-blocking
+3 -1
View File
@@ -49,7 +49,9 @@ fi
[ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
+3 -1
View File
@@ -40,7 +40,9 @@ fi
[ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
+3 -1
View File
@@ -102,7 +102,9 @@ cd "$_REPO_ROOT" || exit 1
PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/<prompt-file-name>"
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
_SID=""
if [ "$_CODEX_MODE" = "resume" ]; then
_SID=$(cat .context/codex-session-id 2>/dev/null)
+3 -1
View File
@@ -93,7 +93,9 @@ cd "$_REPO_ROOT" || exit 1
PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/<prompt-file-name>"
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
_SID=""
if [ "$_CODEX_MODE" = "resume" ]; then
_SID=$(cat .context/codex-session-id 2>/dev/null)
+6 -2
View File
@@ -48,7 +48,9 @@ _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo"
cd "$_REPO_ROOT"
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model review || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model review '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe review || exit $?
# The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST
# and a stall surfaces as a diagnosable exit 124 with an explicit message,
# never as a silent harness kill that downstream reads as "no findings".
@@ -104,7 +106,9 @@ FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/<focus-file-name>"
[ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; }
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
{
printf '%s\n' "IMPORTANT: Do NOT read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. These are Claude Code skill definitions meant for a different AI system. Do not invoke any installed skill (Codex home skills/, .agents/); answer directly. Do NOT modify agents/openai.yaml. Stay focused on repository code only."
+6 -2
View File
@@ -46,7 +46,9 @@ _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo"
cd "$_REPO_ROOT"
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model review || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model review '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe review || exit $?
# The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST
# and a stall surfaces as a diagnosable exit 124 with an explicit message,
# never as a silent harness kill that downstream reads as "no findings".
@@ -95,7 +97,9 @@ FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/<focus-file-name>"
[ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; }
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
_gstack_codex_select_model exec || exit 1
_CODEX_ROLE=''
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
{
printf '%s\n' "IMPORTANT: Do NOT read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. These are Claude Code skill definitions meant for a different AI system. Do not invoke any installed skill (Codex home skills/, .agents/); answer directly. Do NOT modify agents/openai.yaml. Stay focused on repository code only."
+220
View File
@@ -0,0 +1,220 @@
# Model policy
Use a frontier model to challenge a plan, then a smart model to build it.
These are curated recommendations, not a claim that one model wins every task.
User choices take precedence, and a recommendation never switches the running
session or edits the host's native configuration.
| Tier | Default job | Anthropic | OpenAI |
| --- | --- | --- | --- |
| `frontier` | Independent plan review | `claude-fable-5-1` | `gpt-6-astra` |
| `smart` | Implementation recommendation | `claude-opus-5-5` | `gpt-6.1-sol` |
## Inspect and change your policy
This quickstart assumes gstack is installed. It uses no network, credentials or
paid model calls. Run it from the project's repository root in Bash, Git Bash or MSYS:
automated outside reviews use that directory. Inspection from a subdirectory
reflects that subdirectory's native settings and may differ. If your install is vendored
or elsewhere, set `GSTACK_ROOT` to that installed gstack runtime instead.
```bash
if [ -x "$HOME/.claude/skills/gstack/bin/gstack-models" ]; then
GSTACK_ROOT="$HOME/.claude/skills/gstack"
elif [ -x "${CODEX_HOME:-$HOME/.codex}/skills/gstack/bin/gstack-models" ]; then
GSTACK_ROOT="${CODEX_HOME:-$HOME/.codex}/skills/gstack"
else
printf '%s\n' 'Upgrade the gstack install for your host, then rerun this quickstart.' >&2
exit 1
fi
"$GSTACK_ROOT/bin/gstack-models" list
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai
```
The output explains the selected model and which setting won. It also shows
catalog verification metadata. Offline selection does **not** prove that a CLI
is installed, that your account can use the model, or that a vendor's current
catalog has been checked.
Choose the less expensive review tier, then inspect the effective result:
```bash
"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier smart
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai
```
Pin a provider's model within a tier, or reset a pin to the shipped default:
```bash
"$GSTACK_ROOT/bin/gstack-config" set model_frontier_openai gpt-6-astra
"$GSTACK_ROOT/bin/gstack-config" set model_frontier_claude claude-fable-5-1
"$GSTACK_ROOT/bin/gstack-config" unset model_frontier_openai
"$GSTACK_ROOT/bin/gstack-config" unset plan_review_tier
```
To keep using native CLI model settings for plan reviews:
```bash
"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier host
```
Settings apply to the next invocation without rerendering skills. An invocation
already being checked keeps its selected model through execution. Configuration
uses the existing [gstack state root](state-root.md), not a new per-repository
settings file. Invalid writes are rejected and leave the previous value intact;
use `unset`, not an empty value, to restore a default.
## Settings and precedence
| Key | Default | Accepted value |
| --- | --- | --- |
| `plan_review_tier` | `frontier` | `frontier`, `smart`, `host` |
| `implementation_tier` | `smart` | `frontier`, `smart` |
| `model_frontier_claude` | Catalog's Anthropic frontier model | Explicit Anthropic model identifier |
| `model_frontier_openai` | Catalog's OpenAI frontier model | Explicit OpenAI model identifier |
| `model_smart_claude` | Catalog's Anthropic smart model | Explicit Anthropic model identifier |
| `model_smart_openai` | Catalog's OpenAI smart model | Explicit OpenAI model identifier |
For plan reviews, a model named for that invocation wins over
`GSTACK_CLAUDE_MODEL` or `GSTACK_CODEX_MODEL`. That environment override wins over
the selected tier's configured model, which wins over the shipped catalog.
Generic coding-model settings do not silently select a cheaper plan reviewer;
choose `host` mode when that is the behavior you want.
In `host` mode, Codex keeps its existing native-config resolution, including
`CODEX_HOME`, active profiles and the `exec` versus native `review` distinction.
Claude can delegate model selection to its CLI without forcing `--model`; the
inspector reports that delegation rather than inventing an exact model ID.
Actual returned model usage remains separate from the requested model.
Implementation recommendations use a model explicitly requested for that job,
then the configured implementation tier and its provider override, then the
catalog. Outside-review environment overrides do not change implementation
recommendations. Known Claude Code and Codex hosts recommend their own provider;
an unknown or multi-provider host can show both choices.
Custom endpoints and partner platforms need an explicitly compatible model or
`host` mode. Gstack does not guess that public API IDs work with Bedrock, Vertex,
Foundry or a custom Codex provider. A model identifier passing syntax validation
is not proof of account availability. See [provider recovery](troubleshooting.md#model-policy-provider).
Detection is conservative, not a replacement for the CLI's settings engine.
Claude checks environment, user settings, project/shared/local settings, local
settings at the repository and worktree's main checkout, and local managed files.
Codex checks environment, user/active-profile settings and system configuration.
Any detected local custom-route signal requires an explicit model, even if
native trust or precedence rules might suppress it. Server-managed or MDM
policy and extra native CLI flags are not visible to this check; `--help` and
JSON output list these limits. Use an explicit compatible model for those cases,
or `host` mode for plan reviews. Implementation recommendations require an
explicit implementation model rather than delegating to a hidden native choice.
## Which workflows use the tiers?
The plan-review role applies to autoplan's CEO, design, DX and engineering
outside voices; standalone CEO, engineering, developer-experience and design
plan reviews; and the spec quality gate. Existing reasoning effort, access
restrictions, timeouts, review gates and `codex_reviews` consent remain in force.
The host selects the opposing provider independently from the tier. Claude Code
uses the Codex wrapper; Codex uses the Claude Code wrapper. A model's name does
not establish which harness is running.
Manual opposing-model wrappers can opt into `--role plan-review`. Ordinary
code reviews, live design reviews, design consultation, office-hours second
opinions and documentation reviews retain their existing model-selection
behavior. Historical no-role defaults, eval actors/judges, cheap helper models
and setup-time behavioral overlays do not follow this catalog.
Before the first affected paid plan review after upgrade, gstack explains the
independent frontier default and the winning source, with `smart` and `host`
alternatives. Inspection does not mark that notice as seen. Disabled outside
reviews stay disabled; an unavailable model is reported, never silently replaced.
## First completed outside review
Policy setup and a completed review are different milestones. Install and
authenticate the opposing CLI using the [host setup instructions](../README.md#install--about-30-seconds)
before invoking a paid review. From Claude Code, use `/codex challenge --role
plan-review` with your plan. From Codex, use `/claude-code challenge --role
plan-review` (or its installed namespaced name).
Confirm the announced selection, the provider's actual result and the coverage
status. A resolved model, a successful availability probe or a native fallback
is not a completed outside review. An unusable explicit choice needs a repair
at the source named in the error; changing a lower-priority tier cannot override
an environment pin. See [selection recovery](troubleshooting.md#model-policy-selection).
## Machine-readable inspection
`gstack-models` with no arguments is `list`. Both `list` and `resolve` accept
`--json`. Plan-review resolution requires `--provider anthropic|openai`;
implementation resolution can omit the provider to return both recommendations.
```bash
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider anthropic --json
"$GSTACK_ROOT/bin/gstack-models" resolve --role implementation --json
```
JSON output is one document with `schemaVersion: 1`, a `selections` array and
structured errors on failure. Human diagnostics go to stderr. Exit `0` means
valid resolution, including host delegation; `1` means configuration or
selection failure; `2` means invalid usage. Neither inspection nor resolution
writes notice markers, probes a model or fetches vendor sources.
## Keeping shipped recommendations current
The model-policy freshness workflow runs weekly and supports manual dispatch.
It checks official Anthropic and OpenAI recommendation and deprecation sources,
without provider credentials or paid model calls. Normal PR checks use offline
fixtures; a vendor outage does not make unrelated PRs fail.
In GitHub Actions, select **Model-policy freshness** to run it manually. From a
gstack checkout, `bun scripts/model-policy-freshness.ts` performs the same public
source check without changing an issue; unlike `gstack-models`, it uses the
network. The workflow's guarded `--publish` path owns issue updates. Its JSON
report and job summary retain the source evidence.
The report distinguishes `current`, `update-candidate`, `stale` and
`unknown/source-unavailable`. Newer or newly recommended models are candidates,
not automatic replacements. A support promise such as “not sooner than” is not
a scheduled retirement, and an incidental model mention is not a new default.
This evidence covers public API recommendations, not partner-platform entitlement.
One Actions-bot-owned tracking issue preserves the successful check's catalog/source/
parser identity and any unresolved lifecycle findings. Healthy state remains
readable when the issue is closed. A failed or partial check cannot erase a known
retirement, borrow a success from an older catalog, or pretend its evidence is
fresh. A catalog rollback reopens historical findings for a model put back in
use. Only current default-branch runs may update that issue; copied markers in
user-owned issues are not trusted state.
Thirty-day staleness means an absence of trustworthy freshness evidence, not
that a model was released thirty days ago. An unchanged recommendation does not
require a date-only catalog commit or a monthly paid benchmark. The recorded
next-check due date helps maintenance, but a workflow that never runs cannot
alert on its own absence.
### Maintainer update checklist
1. Check the latest freshness run and its due date. Resolve missing, corrupt or
source-unavailable evidence before declaring the catalog current.
2. Verify the official recommendation and lifecycle evidence. Triage announced
retirement/removal within one working day, stale/unavailable evidence within
seven days and discretionary update candidates within thirty days.
3. Open a catalog change with the source URLs, exact model IDs and qualification
evidence, or record why the current default should remain. Never edit users'
personal pins or silently substitute another provider.
4. Confirm CLI model/profile support and tool/reasoning compatibility. Keep the
pinned test runtimes and benchmark rulers stable. A required CLI upgrade is a
separate reviewed change, not an automatic side effect of discovery.
5. Predeclare any budgeted baseline/candidate comparison using existing eval
machinery, fixed prompts/panels and unchanged thresholds. Keep every attempt,
measure useful and unsupported findings, latency and available usage, and do
not treat vendor marketing or a creation timestamp as quality evidence.
6. Pass the affected free checks and selected workflow evaluations, review the
change, and ship it through the normal release process. Only unpinned
role-bearing defaults follow the new shipped catalog.
The workflow does not open an automatic model-upgrade PR, merge changes, switch
active sessions, alter benchmark models or schedule paid evaluations.
+104
View File
@@ -22,6 +22,110 @@ section has a stable link anchor; the reason codes and anchors come from
---
## Model policy
The [model-policy guide](model-policy.md) explains the two tiers, all six
settings, precedence and the read-only inspection command. A resolved selection
is not proof that the requested provider or model can run.
<a id="model-policy-config"></a>
### Model policy configuration is invalid or unreadable
**Meaning.** The new policy path could not read a coherent configuration or a
configured value is invalid. An unreadable file is not treated as an absent
setting: doing that could silently select the premium default.
**Fix.** Repair the file or key named by the error. `plan_review_tier` accepts
`frontier`, `smart` or `host`; `implementation_tier` accepts `frontier` or
`smart`. Use `gstack-config unset <key>` to restore a default, not an empty
value. A rejected `set` leaves the old value intact. Correct ownership/read
permissions rather than deleting an unrelated state directory.
**Expected result.** `gstack-models list` or the corresponding `resolve`
command explains the winning source. No outside model call ran on this error.
<a id="model-policy-provider"></a>
### A custom provider needs an explicit model or host mode
**Meaning.** Native settings or environment flags route the reviewer to a
custom endpoint or partner platform. A public API catalog identifier is not
necessarily a valid deployment identifier there. Malformed native provider
configuration also prevents gstack from assuming the public endpoint.
**Fix.** Supply a compatible model for that request, set the provider's gstack
environment override or tier-model setting, or choose
`gstack-config set plan_review_tier host`. Keep credentials in the provider's
normal authentication mechanism, never in a model setting. If the error names
a malformed native configuration, fix that source first.
**Expected result.** Inspection reports an explicit selection or honest
host-managed delegation. It does not certify account entitlement or send a
public default to an inferred custom endpoint.
<a id="model-policy-selection"></a>
### The selected review model could not be used
**Meaning.** The requested model was rejected or the provider could not
complete the review. A model may be nonexistent, unavailable to the account,
retired, or behind a misconfigured endpoint; preserve the provider's evidence
rather than treating all of these as the same diagnosis. No successful outside
review is implied, and gstack does not substitute a different model.
**Fix.** Change the source named in the error. For example, an environment
override beats a tier setting, so editing the tier cannot replace it. Unset or
correct that override, or explicitly name a supported model for the request.
Follow the existing auth, quota, sandbox or timeout repair when that is the
reported failure. Missing usage identity remains unknown, even if inspection
showed a requested ID.
**Expected result.** The next invocation announces its model and source,
checks that same selection and reports the actual provider result. A cached
probe or a native fallback is not a completed outside review.
<a id="model-policy-freshness"></a>
### Model-catalog freshness is unknown, stale or needs an update
**Meaning.** Official evidence changed, a required source could not be
checked, or the retained evidence does not match this catalog and parser.
`unknown/source-unavailable` is not current. A later source failure must not
erase an earlier retirement finding or close its maintenance issue.
**Fix.** Read the failed source and evidence details in the freshness workflow
report and tracking issue. Repair the source parser or publication access when
needed, then run the workflow manually against the default branch. If a model
was superseded or deprecated, follow the [maintainer checklist](model-policy.md#maintainer-update-checklist)
to qualify a replacement or record a supported mitigation. Do not clear the
issue just because a new release has not been adopted yet.
**Expected result.** Complete source checks produce evidence bound to the
current catalog, while unresolved lifecycle findings remain visible. Personal
pins and unrelated PRs are unchanged. Check the workflow's last run date too:
a scheduled job that never runs cannot report its own absence.
#### Repairing `recoveryRequired` state
This flag deliberately keeps the issue open until a maintainer repairs the
record. Another successful fetch alone cannot certify corrupted history.
1. Save the current issue body, including its retained recovery archive, and
download the latest trustworthy workflow report. Use its `report.state` as
the starting state; do not invent run IDs, catalog hashes or check dates.
2. Reconcile its `lifecycle` array with the issue's independent lifecycle block
and any retained archive. Preserve known retirements, deadlines and human
dispositions. A model currently in the catalog must not keep an old
`resolvedByCatalog` flag from a previous replacement.
3. In the issue editor, restore valid JSON between the `state:begin` and
`state:end` marker lines. Put that same lifecycle array between
`lifecycle:begin` and `lifecycle:end`. Restore the standalone ownership and
region markers, and leave human text outside the owned region intact.
4. Set `recoveryRequired` to `false` only after that reconciliation. If prior
successful evidence cannot be verified, set `lastSuccess` to `null` rather
than making up a fresh success. If lifecycle history cannot be recovered,
leave the flag set and the issue open for investigation.
5. Run **Model-policy freshness** against the default branch. Only complete,
current evidence with no unresolved findings can close the issue. A parsing
or source error leaves it open; do not clear history to manufacture a pass.
## Outside reviews (Codex and Claude Code)
Outside reviews send your diff, plan or question to a second AI provider. Their
+9 -4
View File
@@ -7,7 +7,12 @@ import { spawnSync } from 'node:child_process';
const OLD = 'gstack-claude';
const NEXT = 'gstack-claude-code';
const BANNER = '<!-- AUTO-GENERATED from';
const RUNTIME_FILES = ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts'];
export const CLAUDE_CODE_RUNTIME_FILES = [
'bin/gstack-claude-code', 'bin/gstack-models', 'bin/gstack-config', 'bin/gstack-state-root.sh',
'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts',
'lib/outside-review-result.ts', 'lib/gate-outcomes.ts', 'lib/state-root.ts',
'lib/model-catalog.ts', 'lib/model-policy.ts', 'lib/model-policy-cli.ts', 'lib/model-policy-notice.ts',
];
export interface RenameOptions {
installDir: string;
@@ -171,7 +176,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
(fs.lstatSync(runtime).isSymbolicLink() && !linkIsOurs(runtime, root)) ||
(fs.existsSync(path.join(runtime, 'SKILL.md')) && !generated(path.join(runtime, 'SKILL.md')))
)) throw new Error(`runtime root is not gstack-managed: ${runtime}`);
for (const rel of RUNTIME_FILES) {
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
if (!fs.existsSync(path.join(root, rel))) throw new Error(`replacement runtime is missing ${rel}`);
const parent = path.join(runtime, path.dirname(rel));
if (fs.lstatSync(parent, { throwIfNoEntry: false })?.isSymbolicLink() && !linkIsOurs(parent, root)) {
@@ -190,7 +195,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
throw new Error(`replacement render is not a managed directory: ${canonical}`);
}
copySkill(skill, canonical, root);
for (const rel of RUNTIME_FILES) {
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
const src = path.join(root, rel);
const dst = path.join(runtime, rel);
if (fs.existsSync(dst) && fs.realpathSync(dst) === fs.realpathSync(src)) continue;
@@ -206,7 +211,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
if (!/^name:\s*(?:gstack-)?claude-code\s*$/m.test(fs.readFileSync(path.join(next, 'SKILL.md'), 'utf8'))) {
throw new Error('installed replacement could not be verified');
}
for (const rel of RUNTIME_FILES) fs.accessSync(path.join(runtime, rel), fs.constants.R_OK);
for (const rel of CLAUDE_CODE_RUNTIME_FILES) fs.accessSync(path.join(runtime, rel), fs.constants.R_OK);
// Existing copied workflows must receive the same native host routing as
// the new wrapper, including when setup selected a different host. Only
// refresh installed managed entries; this never installs another host or
+54 -14
View File
@@ -2,6 +2,8 @@
import { spawn, type ChildProcess } from 'node:child_process';
import { resolveClaudeCommand, type ClaudeCommand } from './claude-bin';
import { initializeWindowsReviewJob, WindowsReviewSupervisionError } from './claude-code-windows-job';
import { ModelPolicyError, describeSelection, resolvePlanReviewModel, type ModelSelection } from './model-policy';
import { emitModelPolicyNotice } from './model-policy-notice';
export const CLAUDE_CODE_OUTPUT_LIMIT = 32 * 1024 * 1024;
const DRAIN_TIMEOUT_MS = 500;
@@ -13,6 +15,18 @@ export interface ClaudeCodeOptions {
prompt: string;
resume?: string;
env?: NodeJS.ProcessEnv;
/** Caller-owned workflow role; only plan-review exists. Absent keeps legacy model behavior. */
role?: 'plan-review';
/** A record already resolved for this invocation; it is used as-is and never re-resolved. */
selection?: ModelSelection;
}
export interface ClaudeCodeSelectionSummary {
role: ModelSelection['role'];
tier: ModelSelection['tier'];
status: ModelSelection['status'];
requested_model: string | null;
source: string;
}
export interface ClaudeCodeResult {
@@ -24,6 +38,8 @@ export interface ClaudeCodeResult {
usage?: Record<string, unknown>;
modelUsage?: Record<string, unknown>;
model?: string;
/** What gstack requested, separate from the CLI's reported model/modelUsage. */
selection?: ClaudeCodeSelectionSummary;
exit_code?: number | null;
stderr?: string;
}
@@ -37,7 +53,7 @@ function isObject(value: unknown): value is Record<string, unknown> {
}
/** Keep argument construction separate so wrappers never interpolate a prompt. */
export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resume'>, command: ClaudeCommand, env: NodeJS.ProcessEnv = process.env): string[] {
export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resume' | 'selection'>, command: ClaudeCommand, env: NodeJS.ProcessEnv = process.env): string[] {
const args = [
...command.argsPrefix, '-p', '--output-format', 'json',
'--disable-slash-commands',
@@ -58,9 +74,12 @@ export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resu
+ 'If essential context is missing, identify it explicitly instead of inventing observations.');
}
if (options.access === 'read-only') args.push('--allowedTools', 'Read,Grep,Glob');
// An explicit gstack override wins; otherwise leave native CLI configuration
// and ANTHROPIC_MODEL intact instead of replacing the user's selected model.
if (env.GSTACK_CLAUDE_MODEL) args.push('--model', env.GSTACK_CLAUDE_MODEL);
// A selected record already ranked GSTACK_CLAUDE_MODEL; delegated host mode
// passes no model. One joined argument keeps an ID starting with '-' a value. Without a record, an explicit gstack override wins;
// otherwise leave native CLI configuration and ANTHROPIC_MODEL intact.
if (options.selection) {
if (options.selection.status === 'selected') args.push(`--model=${options.selection.requestedModel}`);
} else if (env.GSTACK_CLAUDE_MODEL) args.push('--model', env.GSTACK_CLAUDE_MODEL);
if (options.resume) args.push('--resume', options.resume);
return args;
}
@@ -127,24 +146,43 @@ export function parseClaudeCodeResult(stdout: string, stderr: string, exitCode:
export async function runClaudeCode(options: ClaudeCodeOptions): Promise<ClaudeCodeResult> {
if (!options.cwd || !['none', 'read-only'].includes(options.access) ||
!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0 || options.timeoutMs > 2_147_483_647 ||
!options.prompt.trim() || (options.resume !== undefined && !options.resume.trim())) {
!options.prompt.trim() || (options.resume !== undefined && !options.resume.trim()) ||
(options.role !== undefined && options.role !== 'plan-review')) {
return failure('arguments', 'Provide --cwd, --access none|read-only, a positive --timeout-ms, and a nonempty prompt on stdin.');
}
const env = options.env ?? process.env;
const command = resolveClaudeCommand(env);
if (!command) return failure('not-found', 'Claude Code CLI not found. Install Claude Code or set GSTACK_CLAUDE_BIN, then retry.', true);
let selection = options.selection;
if (!selection && options.role === 'plan-review') {
try {
selection = resolvePlanReviewModel({ provider: 'anthropic', env, cwd: options.cwd });
} catch (error) {
if (!(error instanceof ModelPolicyError)) throw error;
return failure(error.reason, `${error.problem} ${error.cause} Repair: ${error.repair.join('; ')}. ${error.docs} No Claude Code call was made; outside coverage is unavailable.`, true);
}
}
if (selection) {
process.stderr.write(`CLAUDE_MODEL: ${describeSelection(selection)}\n`);
if (selection.role === 'plan-review') emitModelPolicyNotice(selection, { env });
}
const summary: ClaudeCodeSelectionSummary | undefined = selection && {
role: selection.role, tier: selection.tier, status: selection.status,
requested_model: selection.requestedModel, source: selection.source.label,
};
const withSelection = (result: ClaudeCodeResult): ClaudeCodeResult => (summary ? { ...result, selection: summary } : result);
let child: ChildProcess;
try {
child = spawn(command.command, claudeCodeArgs(options, command, env), {
child = spawn(command.command, claudeCodeArgs({ ...options, selection }, command, env), {
cwd: options.cwd, env, stdio: ['pipe', 'pipe', 'pipe'],
detached: process.platform !== 'win32', windowsHide: true,
});
} catch (error) {
return failure('spawn', `Claude Code could not start: ${(error as Error).message}`, true);
return withSelection(failure('spawn', `Claude Code could not start: ${(error as Error).message}`, true));
}
return await new Promise<ClaudeCodeResult>((resolve) => {
return withSelection(await new Promise<ClaudeCodeResult>((resolve) => {
let settled = false;
let stopped: ClaudeCodeResult | undefined;
let exitCode: number | null = null;
@@ -222,7 +260,7 @@ export async function runClaudeCode(options: ClaudeCodeOptions): Promise<ClaudeC
}
});
child.stdin?.end(options.prompt);
});
}));
}
export async function claudeCodeMain(argv: string[]): Promise<number> {
@@ -230,22 +268,24 @@ export async function claudeCodeMain(argv: string[]): Promise<number> {
try {
const values = new Map<string, string>();
for (let i = 0; i < argv.length; i += 2) {
if (!['--cwd', '--access', '--timeout-ms', '--resume'].includes(argv[i]) || values.has(argv[i]) || argv[i + 1] === undefined) {
throw new Error('Usage: gstack-claude-code --cwd <repo> --access none|read-only --timeout-ms <n> [--resume <session-id>] (prompt on stdin)');
if (!['--cwd', '--access', '--timeout-ms', '--resume', '--role'].includes(argv[i]) || values.has(argv[i]) || argv[i + 1] === undefined) {
throw new Error('Usage: gstack-claude-code --cwd <repo> --access none|read-only --timeout-ms <n> [--resume <session-id>] [--role plan-review] (prompt on stdin)');
}
values.set(argv[i], argv[i + 1]);
}
const cwd = values.get('--cwd') ?? '';
const access = values.get('--access') as ClaudeCodeOptions['access'];
const timeoutMs = Number(values.get('--timeout-ms'));
if (!cwd || !['none', 'read-only'].includes(access) || !Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647) {
throw new Error('Provide --cwd, --access none|read-only, and a positive --timeout-ms.');
const role = values.get('--role');
if (!cwd || !['none', 'read-only'].includes(access) || !Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647 ||
(role !== undefined && role !== 'plan-review')) {
throw new Error('Provide --cwd, --access none|read-only, a positive --timeout-ms, and optionally --role plan-review.');
}
// This entry runs in a dedicated process. Its Windows job owns the CLI and
// descendants even after the immediate provider process exits; the final
// process.exit happens only after the result below has flushed to stdout.
await initializeWindowsReviewJob();
result = await runClaudeCode({ cwd, access, timeoutMs, resume: values.get('--resume'), prompt: await Bun.stdin.text() });
result = await runClaudeCode({ cwd, access, timeoutMs, resume: values.get('--resume'), role: role as ClaudeCodeOptions['role'], prompt: await Bun.stdin.text() });
} catch (error) {
result = error instanceof WindowsReviewSupervisionError
? failure('supervision', error.message, true)
+43
View File
@@ -0,0 +1,43 @@
import { createHash } from 'node:crypto';
export type ModelProvider = 'anthropic' | 'openai';
export type ModelTier = 'frontier' | 'smart';
export const MODEL_PROVIDERS: readonly ModelProvider[] = Object.freeze(['anthropic', 'openai'] as const);
export const MODEL_TIERS: readonly ModelTier[] = Object.freeze(['frontier', 'smart'] as const);
export const MODEL_CATALOG_VERSION = 1 as const;
export interface ModelCatalogEntry {
readonly provider: ModelProvider;
readonly tier: ModelTier;
readonly model: string;
readonly sourceUrl: string;
readonly verifiedAt: string;
}
const ANTHROPIC_SOURCE = 'https://platform.claude.com/docs/en/models/overview';
const OPENAI_SOURCE = 'https://developers.openai.com/api/docs/models';
export const MODEL_CATALOG: readonly ModelCatalogEntry[] = Object.freeze([
{ provider: 'anthropic', tier: 'frontier', model: 'claude-fable-5-1', sourceUrl: ANTHROPIC_SOURCE, verifiedAt: '2026-10-07' },
{ provider: 'openai', tier: 'frontier', model: 'gpt-6-astra', sourceUrl: OPENAI_SOURCE, verifiedAt: '2026-10-07' },
{ provider: 'anthropic', tier: 'smart', model: 'claude-opus-5-5', sourceUrl: ANTHROPIC_SOURCE, verifiedAt: '2026-10-07' },
{ provider: 'openai', tier: 'smart', model: 'gpt-6.1-sol', sourceUrl: OPENAI_SOURCE, verifiedAt: '2026-10-07' },
].map(entry => Object.freeze(entry as ModelCatalogEntry)));
export function catalogEntry(provider: ModelProvider, tier: ModelTier): ModelCatalogEntry {
const entry = MODEL_CATALOG.find(e => e.provider === provider && e.tier === tier);
if (!entry) throw new Error(`model catalog has no ${tier} entry for ${provider}`);
return entry;
}
export function modelCatalogCanonicalJson(): string {
return JSON.stringify({
version: MODEL_CATALOG_VERSION,
entries: MODEL_CATALOG.map(e => ({ provider: e.provider, tier: e.tier, model: e.model, sourceUrl: e.sourceUrl, verifiedAt: e.verifiedAt })),
});
}
export function modelCatalogSha256(): string {
return createHash('sha256').update(modelCatalogCanonicalJson()).digest('hex');
}
+254
View File
@@ -0,0 +1,254 @@
import * as path from 'node:path';
import { MODEL_CATALOG, MODEL_CATALOG_VERSION, MODEL_PROVIDERS, MODEL_TIERS, modelCatalogSha256, type ModelProvider } from './model-catalog';
import {
CUSTOM_PROVIDER_UNSCANNED,
describeSelection,
MODEL_OVERRIDE_KEYS,
ModelPolicyError,
modelPolicyCommands,
readModelPolicyConfig,
resolveImplementationModels,
resolvePlanReviewModel,
type ModelPolicyConfig,
type ModelPolicyErrorJson,
type ModelRole,
type ModelSelection,
} from './model-policy';
import type { StateRootEnv } from './state-root';
export interface ModelsCliResult {
code: 0 | 1 | 2;
stdout: string;
stderr: string;
}
type CliError = (ModelPolicyErrorJson | { reason: 'usage'; problem: string; cause: string; key: null; source: null; repair: string[]; docs: string }) & { role?: ModelRole; provider?: ModelProvider };
const OFFLINE_NOTE = 'offline: no network, no paid model call and no writes; this does not prove account entitlement, CLI availability or current vendor status';
const DOCS = 'https://github.com/garrytan/gstack/blob/main/docs/model-policy.md';
const CONFIG_BIN = path.join(import.meta.dir, '..', 'bin', 'gstack-config');
export const MODELS_HELP = `Usage:
gstack-models [list] [--json]
gstack-models resolve --role plan-review --provider anthropic|openai [--json]
gstack-models resolve --role implementation [--provider anthropic|openai] [--json]
gstack-models --help
Shows which model each gstack role uses and why. It reads config.yaml, the
GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL environment variables and native
provider settings, then exits. It is offline: no network, no paid model call,
no writes. A resolved model is not proof that your account can use it, that
the CLI is installed, or that the vendor still offers it.
Roles:
plan-review The opposing-provider reviewer for plan reviews. Default tier
frontier. Precedence: explicit request, then
GSTACK_CLAUDE_MODEL (anthropic) or GSTACK_CODEX_MODEL (openai),
then model_<tier>_<claude|openai>, then the shipped catalog.
plan_review_tier=host delegates to the host CLI's own settings:
the model is host-controlled and unknown until the review runs.
implementation The model gstack recommends for implementing an approved plan.
Default tier smart. Precedence: explicit request, then
model_<tier>_<claude|openai>, then the shipped catalog.
Environment overrides do not apply. Without --provider both
provider recommendations are shown. gstack never switches your
session's model.
Settings (gstack-config set|unset <key>):
plan_review_tier frontier | smart | host (default frontier)
implementation_tier frontier | smart (default smart)
model_frontier_claude, model_frontier_openai,
model_smart_claude, model_smart_openai (default: shipped catalog ID)
A custom Anthropic or OpenAI endpoint (Bedrock, Vertex, Foundry, a base-URL
override or a Codex model_provider) never receives a shipped public ID: set a
per-tier model, an environment override (plan-review only) or host mode.
Detection reads the environment, the env block of Claude's user, project
(.claude/settings.json, .claude/settings.local.json in the working directory
and repository root) and managed settings files, and Codex's user and system
config.toml. It cannot see server-managed or MDM policy, claude --settings, or
codex -c/--profile flags: if you route those to a custom endpoint, set a
per-tier model or host mode.
Output: --json prints one JSON document (schemaVersion 1) with a selections
array and structured errors. Human diagnostics go to stderr.
Exit: 0 valid resolution (including host delegation), 1 config or selection
failure, 2 invalid usage.
Docs: ${DOCS}
`;
function usage(problem: string, json: boolean): ModelsCliResult {
const error: CliError = { reason: 'usage', problem, cause: 'see gstack-models --help', key: null, source: null, repair: ['gstack-models --help'], docs: DOCS };
if (json) return { code: 2, stdout: `${JSON.stringify(document(null, null, [], [error]), null, 2)}\n`, stderr: '' };
return { code: 2, stdout: '', stderr: `gstack-models: ${problem}\n${MODELS_HELP.split('\n\n')[0]}\n` };
}
function parseProvider(value: string | undefined): ModelProvider | null {
return value === 'anthropic' || value === 'openai' ? value : null;
}
function document(command: 'list' | 'resolve' | null, config: ModelPolicyConfig | null, selections: ModelSelection[], errors: CliError[]) {
return {
schemaVersion: 1,
command,
ok: errors.length === 0,
offline: true,
note: OFFLINE_NOTE,
configPath: config?.path ?? null,
catalog: { version: MODEL_CATALOG_VERSION, sha256: modelCatalogSha256(), entries: MODEL_CATALOG },
settings: config ? settingsOf(config) : null,
tiers: config ? tiersOf(config) : null,
providerDetection: { scope: 'declared local settings files and environment only', unscanned: CUSTOM_PROVIDER_UNSCANNED },
selections,
errors,
};
}
function settingsOf(config: ModelPolicyConfig) {
const settings: Record<string, { value: string | null; effective: string; origin: 'config' | 'default' }> = {
plan_review_tier: { value: config.planReviewTier.origin === 'config' ? config.planReviewTier.value : null, effective: config.planReviewTier.value, origin: config.planReviewTier.origin },
implementation_tier: { value: config.implementationTier.origin === 'config' ? config.implementationTier.value : null, effective: config.implementationTier.value, origin: config.implementationTier.origin },
};
for (const entry of MODEL_CATALOG) {
const override = config.overrides[entry.provider][entry.tier];
settings[MODEL_OVERRIDE_KEYS[entry.provider][entry.tier]] = { value: override, effective: override ?? entry.model, origin: override ? 'config' : 'default' };
}
return settings;
}
function tiersOf(config: ModelPolicyConfig) {
return MODEL_CATALOG.map(entry => ({
tier: entry.tier,
provider: entry.provider,
catalogModel: entry.model,
verifiedAt: entry.verifiedAt,
sourceUrl: entry.sourceUrl,
overrideKey: MODEL_OVERRIDE_KEYS[entry.provider][entry.tier],
override: config.overrides[entry.provider][entry.tier],
configuredModel: config.overrides[entry.provider][entry.tier] ?? entry.model,
}));
}
function attempt(role: ModelRole, provider: ModelProvider, run: () => ModelSelection[], selections: ModelSelection[], errors: CliError[]): void {
try {
selections.push(...run());
} catch (error) {
if (!(error instanceof ModelPolicyError)) throw error;
errors.push({ ...error.toJSON(), role, provider });
}
}
function renderError(error: CliError): string {
const scope = error.role ? ` (${error.role}${error.provider ? ` via ${error.provider}` : ''})` : '';
return [
`gstack-models: ${error.problem}${scope}`,
` cause: ${error.cause}`,
...(error.key ? [` source: ${error.key}`] : []),
...error.repair.map(step => ` fix: ${step}`),
` docs: ${error.docs}`,
].join('\n') + '\n';
}
function renderSelections(role: ModelRole, config: ModelPolicyConfig, selections: ModelSelection[]): string[] {
const choice = role === 'plan-review' ? config.planReviewTier : config.implementationTier;
const purpose = role === 'plan-review' ? 'opposing-provider plan evaluation' : 'recommendation only; gstack never switches your session';
const lines = [`${role}: tier ${choice.value} (${choice.origin === 'config' ? `${choice.key} in ${choice.path}` : `default; set ${choice.key}`}) - ${purpose}`];
for (const selection of selections.filter(s => s.role === role)) {
lines.push(` ${describeSelection(selection)}`);
}
return lines;
}
function renderList(config: ModelPolicyConfig, selections: ModelSelection[]): string {
const commands = modelPolicyCommands();
const lines = [
`gstack model policy (${OFFLINE_NOTE})`,
`config: ${config.path}${config.exists ? '' : ' (not created yet; defaults apply)'}`,
'',
...renderSelections('plan-review', config, selections),
...renderSelections('implementation', config, selections),
'',
`catalog v${MODEL_CATALOG_VERSION} (sha256 ${modelCatalogSha256().slice(0, 12)})`,
];
for (const tier of MODEL_TIERS) {
for (const provider of MODEL_PROVIDERS) {
const entry = MODEL_CATALOG.find(e => e.tier === tier && e.provider === provider)!;
const override = config.overrides[provider][tier];
lines.push(` ${tier.padEnd(8)} ${provider.padEnd(9)} ${entry.model.padEnd(18)} verified ${entry.verifiedAt} ${MODEL_OVERRIDE_KEYS[provider][tier]}: ${override ?? 'unset'}`);
}
}
lines.push(
'',
'change (gstack-config is ' + CONFIG_BIN + '):',
` ${commands.useSmart}`,
` ${commands.useHost}`,
` ${commands.restoreDefault}`,
` ${commands.overrideExample('openai', 'frontier')}`,
` ${commands.unsetOverride('openai', 'frontier')}`,
'environment overrides GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL outrank plan-review tier settings',
'custom-endpoint detection reads local settings files only; server-managed/MDM policy and CLI flags are not visible (see --help)',
);
return lines.join('\n') + '\n';
}
function renderResolve(role: ModelRole, config: ModelPolicyConfig, selections: ModelSelection[]): string {
const lines = renderSelections(role, config, selections);
if (selections.some(s => s.status === 'delegated-host')) {
lines.push(' host-controlled: gstack passes no model; the actual model is reported after the review runs');
}
lines.push(`(${OFFLINE_NOTE})`);
return lines.join('\n') + '\n';
}
export function modelsMain(args: string[], env: StateRootEnv = process.env, platform: NodeJS.Platform = process.platform, cwd: string = process.cwd()): ModelsCliResult {
const json = args.includes('--json');
if (args.includes('--help') || args.includes('-h')) return { code: 0, stdout: MODELS_HELP, stderr: '' };
const rest = [...args];
const command = rest[0] === undefined || rest[0].startsWith('--') ? 'list' : rest.shift()!;
if (command !== 'list' && command !== 'resolve') return usage(`unknown command '${command}'`, json);
const options: Record<string, string> = {};
while (rest.length) {
const arg = rest.shift()!;
if (arg === '--json') continue;
const [name, inline] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, undefined];
if (command === 'list' || (name !== '--role' && name !== '--provider')) return usage(`unknown argument '${arg}' for ${command}`, json);
if (options[name] !== undefined) return usage(`${name} given twice`, json);
const value = inline ?? rest.shift();
if (value === undefined || value === '' || value.startsWith('--')) return usage(`${name} needs a value`, json);
options[name] = value;
}
const role = options['--role'];
const providerArg = options['--provider'];
const provider = parseProvider(providerArg);
if (command === 'resolve') {
if (role !== 'plan-review' && role !== 'implementation') return usage(role === undefined ? 'resolve needs --role plan-review|implementation' : `unknown role '${role}': use plan-review or implementation`, json);
if (providerArg !== undefined && !provider) return usage(`unknown provider '${providerArg}': use anthropic or openai`, json);
if (role === 'plan-review' && !provider) return usage('resolve --role plan-review needs --provider anthropic|openai (the opposing provider for your host)', json);
}
let config: ModelPolicyConfig;
try {
config = readModelPolicyConfig({ env, platform });
} catch (error) {
if (!(error instanceof ModelPolicyError)) throw error;
const errors: CliError[] = [error.toJSON()];
return json
? { code: 1, stdout: `${JSON.stringify(document(command, null, [], errors), null, 2)}\n`, stderr: '' }
: { code: 1, stdout: '', stderr: errors.map(renderError).join('') };
}
const selections: ModelSelection[] = [];
const errors: CliError[] = [];
const roles: ModelRole[] = command === 'list' ? ['plan-review', 'implementation'] : [role as ModelRole];
for (const r of roles) {
for (const p of provider ? [provider] : MODEL_PROVIDERS) {
attempt(r, p, () => r === 'plan-review'
? [resolvePlanReviewModel({ provider: p, env, platform, cwd, config })]
: resolveImplementationModels({ provider: p, env, platform, cwd, config }), selections, errors);
}
}
const code = errors.length ? 1 : 0;
if (json) return { code, stdout: `${JSON.stringify(document(command, config, selections, errors), null, 2)}\n`, stderr: '' };
const stdout = command === 'list' ? renderList(config, selections) : renderResolve(role as ModelRole, config, selections);
return { code, stdout, stderr: errors.map(renderError).join('') };
}
+53
View File
@@ -0,0 +1,53 @@
import { readBoundedResponse, sha256, type Fetcher } from './model-policy-freshness-sources';
import { FRESHNESS_ISSUE_AUTHOR, type FreshnessGitHub, type TrackingIssue } from './model-policy-freshness-publication';
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
function issueFromJson(value: unknown): TrackingIssue {
if (!object(value) || !Number.isSafeInteger(value.number) || Number(value.number) <= 0 || (value.body !== null && typeof value.body !== 'string') || !['open', 'closed'].includes(String(value.state)) || !object(value.user) || typeof value.user.login !== 'string') throw new Error('github-invalid-issue');
return { number: Number(value.number), body: value.body === null ? '' : value.body as string, state: value.state as 'open' | 'closed', author: value.user.login };
}
export function freshnessGitHub(repository: string, token: string, fetcher: Fetcher = fetch): FreshnessGitHub {
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository) || !token) throw new Error('github-publication-prerequisites-missing');
const root = `https://api.github.com/repos/${repository}`;
async function request(path: string, signal: AbortSignal, method = 'GET', body?: unknown): Promise<unknown> {
if (signal.aborted) throw new Error('github-publication-aborted');
const response = await fetcher(`${root}${path}`, {
method, signal, redirect: 'error',
headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'Content-Type': 'application/json' },
...(body === undefined ? {} : { body: JSON.stringify(body) }),
});
if (!response.ok) { await response.body?.cancel(); throw new Error(`github-http-${response.status}`); }
try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(await readBoundedResponse(response))); }
catch { throw new Error('github-invalid-or-oversized-response'); }
}
return {
async defaultCatalog(signal) {
const repo = await request('', signal);
if (!object(repo) || typeof repo.default_branch !== 'string' || !repo.default_branch) throw new Error('github-default-branch-unavailable');
const file = await request(`/contents/lib/model-catalog.ts?ref=${encodeURIComponent(repo.default_branch)}`, signal);
if (!object(file) || file.encoding !== 'base64' || typeof file.content !== 'string' || typeof file.size !== 'number' || file.size <= 0 || file.size > 2 * 1024 * 1024 || !/^[A-Za-z0-9+/=\n\r]+$/.test(file.content)) throw new Error('github-default-catalog-unavailable');
const bytes = Buffer.from(file.content, 'base64');
if (bytes.length !== file.size) throw new Error('github-default-catalog-size-mismatch');
return { defaultBranch: repo.default_branch, sourceSha256: sha256(bytes) };
},
async listIssues(signal) {
const issues: TrackingIssue[] = [];
for (let page = 1; page <= 100; page++) {
const result = await request(`/issues?state=all&creator=${encodeURIComponent(FRESHNESS_ISSUE_AUTHOR)}&per_page=100&page=${page}`, signal);
if (!Array.isArray(result)) throw new Error('github-issue-list-unavailable');
for (const value of result) {
const issue = issueFromJson(value);
if (issue.author !== FRESHNESS_ISSUE_AUTHOR) throw new Error('github-issue-owner-filter-mismatch');
if (object(value) && !value.pull_request) issues.push(issue);
}
if (result.length < 100) return issues;
}
throw new Error('github-issue-pagination-limit');
},
async readIssue(number, signal) { return issueFromJson(await request(`/issues/${number}`, signal)); },
async createIssue(body, signal) { return issueFromJson(await request('/issues', signal, 'POST', { title: 'Model-policy freshness: advisory maintenance', body })); },
async updateIssue(number, update, signal) { await request(`/issues/${number}`, signal, 'PATCH', update); },
};
}
@@ -0,0 +1,6 @@
import { parseOfficialSource, type SourceId } from './model-policy-freshness-sources';
self.onmessage = (event: MessageEvent<{ id: SourceId; text: string }>) => {
try { self.postMessage({ ok: true, parsed: parseOfficialSource(event.data.id, event.data.text) }); }
catch (error) { self.postMessage({ ok: false, error: error instanceof Error ? error.message : 'parser-unavailable' }); }
};
+127
View File
@@ -0,0 +1,127 @@
import { FRESHNESS_BOUNDS, withinDeadline } from './model-policy-freshness-sources';
import { assessFreshness, observationIsOlder, renderFreshnessReport, sameCatalog, validLifecycle, validState, type CatalogIdentity, type FreshnessObservation, type FreshnessReport, type FreshnessState, type LifecycleFinding } from './model-policy-freshness';
export const FRESHNESS_ISSUE_MARKER = '<!-- gstack:model-policy-freshness:v1 -->';
export const FRESHNESS_REGION_BEGIN = '<!-- gstack:model-policy-freshness:begin -->';
export const FRESHNESS_REGION_END = '<!-- gstack:model-policy-freshness:end -->';
const STATE_BEGIN = '<!-- gstack:model-policy-freshness:state:begin -->';
const STATE_END = '<!-- gstack:model-policy-freshness:state:end -->';
const HISTORY_BEGIN = '<!-- gstack:model-policy-freshness:lifecycle:begin -->';
const HISTORY_END = '<!-- gstack:model-policy-freshness:lifecycle:end -->';
export const FRESHNESS_ISSUE_AUTHOR = 'github-actions[bot]';
export type TrackingIssue = { number: number; body: string; state: 'open' | 'closed'; author: string };
export type FreshnessGitHub = {
defaultCatalog: (signal: AbortSignal) => Promise<{ defaultBranch: string; sourceSha256: string }>;
listIssues: (signal: AbortSignal) => Promise<TrackingIssue[]>;
readIssue: (number: number, signal: AbortSignal) => Promise<TrackingIssue>;
createIssue: (body: string, signal: AbortSignal) => Promise<TrackingIssue>;
updateIssue: (number: number, update: { body: string; state: 'open' | 'closed' }, signal: AbortSignal) => Promise<void>;
};
export type StoredIssue = { state: FreshnessState | null; lifecycle: LifecycleFinding[]; recoveryRequired: boolean; problem: string | null };
const KNOWN_MARKERS = [FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, STATE_BEGIN, STATE_END, HISTORY_BEGIN, HISTORY_END];
function markerLines(body: string, marker?: string): { start: number; end: number; text: string }[] {
return [...body.matchAll(/[^\r\n]+/g)].flatMap(line => {
const text = line[0];
const matches = marker === undefined ? /^[\uFEFF \t]*(?:<!--|<!-)[ \t]*gstack:model-policy-freshness/i.test(text) : text === marker;
return matches ? [{ start: line.index, end: line.index + text.length, text }] : [];
});
}
const occurrences = (body: string, marker: string): number => markerLines(body, marker).length;
const malformedMarkers = (body: string): boolean => markerLines(body).some(line => !KNOWN_MARKERS.includes(line.text));
function block(body: string, begin: string, end: string): { start: number; end: number; text: string } | null {
const starts = markerLines(body, begin);
const ends = markerLines(body, end);
if (starts.length !== 1 || ends.length !== 1 || ends[0].start <= starts[0].end) return null;
return { start: starts[0].start, end: ends[0].end, text: body.slice(starts[0].end, ends[0].start).trim() };
}
function jsonBlock(body: string, begin: string, end: string): unknown {
const text = block(body, begin, end)?.text;
if (!text) return null;
try { return JSON.parse(text); } catch { return null; }
}
export function readFreshnessIssue(body: string): StoredIssue {
const owned = block(body, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END);
const history = jsonBlock(owned?.text ?? body, HISTORY_BEGIN, HISTORY_END);
const state = jsonBlock(owned?.text ?? body, STATE_BEGIN, STATE_END);
const lifecycle = validLifecycle(history) ? history : validState(state) ? state.lifecycle : [];
if (owned && occurrences(body, FRESHNESS_ISSUE_MARKER) === 1 && !malformedMarkers(body) && validState(state) && validLifecycle(history) && JSON.stringify(state.lifecycle) === JSON.stringify(history)) return { state, lifecycle, recoveryRequired: state.recoveryRequired, problem: null };
return { state: validState(state) ? state : null, lifecycle, recoveryRequired: true, problem: 'missing-or-corrupt-authoritative-state:retain-history-and-repair-manually' };
}
export function writeFreshnessIssue(body: string, report: FreshnessReport): string {
const region = [FRESHNESS_REGION_BEGIN, renderFreshnessReport(report), '', STATE_BEGIN, JSON.stringify(report.state), STATE_END, '', HISTORY_BEGIN, JSON.stringify(report.state.lifecycle), HISTORY_END, FRESHNESS_REGION_END].join('\n');
let next: string;
const owned = block(body, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END);
if (malformedMarkers(body)) throw new Error('corrupt-owned-region:manual-recovery-required');
if (owned !== null) {
const recovery = readFreshnessIssue(body);
const archive = recovery.problem ? `\n\n<details><summary>Corrupt authoritative region retained for manual recovery (base64 UTF-8)</summary>\n\n\`\`\`text\n${Buffer.from(owned.text, 'utf8').toString('base64')}\n\`\`\`\n</details>` : '';
next = body.slice(0, owned.start) + region + archive + body.slice(owned.end);
} else if (markerLines(body).some(line => line.text !== FRESHNESS_ISSUE_MARKER)) {
throw new Error('corrupt-owned-region:manual-recovery-required');
} else {
next = `${body}${body ? '\n\n' : ''}${region}`;
}
const topMarkers = occurrences(body, FRESHNESS_ISSUE_MARKER);
if (topMarkers > 1) throw new Error('duplicate-tracking-marker:manual-recovery-required');
if (!topMarkers) {
if (markerLines(body).length) throw new Error('corrupt-tracking-marker:manual-recovery-required');
next = `${FRESHNESS_ISSUE_MARKER}\n${next}`;
}
if (new TextEncoder().encode(next).length > FRESHNESS_BOUNDS.issueBytes) throw new Error('tracking-issue-size-limit:manual-triage-required');
return next;
}
export type PublicationResult = { action: 'report-only' | 'rejected-obsolete-catalog' | 'rejected-older-observation' | 'updated' | 'created'; issueNumber?: number; report?: FreshnessReport };
async function checkAuthority(github: FreshnessGitHub, catalog: CatalogIdentity, signal: AbortSignal): Promise<boolean> {
const current = await github.defaultCatalog(signal);
if (signal.aborted) throw new Error('publication-aborted');
if (current.defaultBranch !== 'main') throw new Error('default-branch-is-not-main:publication-disabled');
return catalog.sourceSha256 === current.sourceSha256;
}
function publicationReport(observation: FreshnessObservation, issue?: TrackingIssue): FreshnessReport | null {
const stored = issue ? readFreshnessIssue(issue.body) : { state: null, lifecycle: [], recoveryRequired: false };
if (stored.state && observationIsOlder(observation, stored.state.latest)) return null;
if (stored.lifecycle.some(finding => Date.parse(finding.lastObservedAt) > Date.parse(observation.checkedAt))) return null;
const report = assessFreshness(observation, stored.state, stored.lifecycle, stored.recoveryRequired);
if (report.state.lastSuccess && !sameCatalog(observation.catalog, report.state.lastSuccess.catalog) && report.status === 'current') throw new Error('mismatched-evidence');
return report;
}
export async function publishFreshness(observation: FreshnessObservation, github: FreshnessGitHub, ref: string): Promise<PublicationResult> {
if (ref !== 'refs/heads/main') return { action: 'report-only', report: assessFreshness(observation) };
const controller = new AbortController();
const signal = controller.signal;
const publish = async (): Promise<PublicationResult> => {
if (!await checkAuthority(github, observation.catalog, signal)) return { action: 'rejected-obsolete-catalog' };
const issues = (await github.listIssues(signal)).filter(issue => markerLines(issue.body).length > 0);
if (issues.some(issue => issue.author !== FRESHNESS_ISSUE_AUTHOR)) throw new Error('untrusted-tracking-marker-owner:manual-recovery-required');
if (issues.length > 1 || issues.some(issue => occurrences(issue.body, FRESHNESS_ISSUE_MARKER) > 1)) throw new Error('duplicate-tracking-marker:manual-recovery-required');
if (issues.some(issue => occurrences(issue.body, FRESHNESS_ISSUE_MARKER) !== 1 || malformedMarkers(issue.body))) throw new Error('corrupt-tracking-marker:manual-recovery-required');
let issue = issues[0];
if (!await checkAuthority(github, observation.catalog, signal)) return { action: 'rejected-obsolete-catalog' };
if (issue) issue = await github.readIssue(issue.number, signal);
if (issue && (occurrences(issue.body, FRESHNESS_ISSUE_MARKER) !== 1 || malformedMarkers(issue.body) || issue.author !== FRESHNESS_ISSUE_AUTHOR)) throw new Error('tracking-marker-or-owner-changed');
const report = publicationReport(observation, issue);
if (!report) return { action: 'rejected-older-observation', issueNumber: issue.number };
const body = writeFreshnessIssue(issue?.body ?? '', report);
const state = report.status === 'current' && report.complete && !report.state.lifecycle.some(finding => !finding.resolvedByCatalog) ? 'closed' : 'open';
if (signal.aborted) throw new Error('publication-aborted');
if (!issue) {
issue = await github.createIssue(body, signal);
if (issue.author !== FRESHNESS_ISSUE_AUTHOR) throw new Error('created-issue-owner-mismatch:manual-recovery-required');
if (state === 'closed') await github.updateIssue(issue.number, { body, state }, signal);
return { action: 'created', issueNumber: issue.number, report };
}
await github.updateIssue(issue.number, { body, state }, signal);
return { action: 'updated', issueNumber: issue.number, report };
};
return withinDeadline(publish(), FRESHNESS_BOUNDS.publicationMs, controller);
}
+232
View File
@@ -0,0 +1,232 @@
import { createHash } from 'node:crypto';
import { marked, type Tokens } from 'marked';
export const FRESHNESS_SOURCES = [
{ id: 'anthropic-models', url: 'https://platform.claude.com/docs/en/models/overview.md' },
{ id: 'anthropic-lifecycle', url: 'https://platform.claude.com/docs/en/about-claude/model-deprecations.md' },
{ id: 'openai-models', url: 'https://developers.openai.com/api/docs/models.md' },
{ id: 'openai-lifecycle', url: 'https://developers.openai.com/api/docs/deprecations.md' },
] as const;
export type SourceId = typeof FRESHNESS_SOURCES[number]['id'];
export const FRESHNESS_BOUNDS = { responseMs: 30_000, responseBytes: 2 * 1024 * 1024, redirects: 3, checkMs: 180_000, publicationMs: 60_000, issueBytes: 60_000 } as const;
export type Fetcher = (url: string, init: RequestInit) => Promise<Response>;
export type LifecycleRow = { modelId: string; state: 'active' | 'legacy' | 'deprecated' | 'removed'; deadline: string | null; supportFloor: string | null };
export type ParsedSource = { recommendations?: { frontier: string; smart: string }; lineup?: string[]; lifecycle?: LifecycleRow[] };
export type SourceReceipt = { id: SourceId; url: string; finalUrl: string; sha256: string; bytes: number; parsed: ParsedSource };
export type SourceResult = { id: SourceId; receipt: SourceReceipt; error?: never } | { id: SourceId; error: string; receipt?: never };
export const sha256 = (text: string | Uint8Array): string => createHash('sha256').update(text).digest('hex');
export async function withinDeadline<T>(task: Promise<T>, milliseconds: number, controller?: AbortController): Promise<T> {
let timer: ReturnType<typeof setTimeout>;
const expiry = new Promise<never>((_, reject) => {
timer = setTimeout(() => { controller?.abort(); reject(new Error('deadline-exceeded')); }, milliseconds);
});
try { return await Promise.race([task, expiry]); } finally { clearTimeout(timer!); }
}
function allowedUrl(value: string): boolean {
return FRESHNESS_SOURCES.some(source => source.url === value)
|| value === 'https://developers.openai.com/api/docs/deprecations';
}
export async function readBoundedResponse(response: Response): Promise<Uint8Array> {
if (Number(response.headers.get('content-length')) > FRESHNESS_BOUNDS.responseBytes) {
await response.body?.cancel(); throw new Error('response-too-large');
}
if (!response.body) throw new Error('empty-response');
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let bytes = 0;
try {
while (true) {
const chunk = await reader.read();
if (chunk.done) break;
bytes += chunk.value.length;
if (bytes > FRESHNESS_BOUNDS.responseBytes) throw new Error('response-too-large');
chunks.push(chunk.value);
}
} finally { await reader.cancel().catch(() => {}); }
const data = new Uint8Array(bytes);
let offset = 0;
for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.length; }
return data;
}
export async function fetchOfficialSource(id: SourceId, fetcher: Fetcher = fetch, parentSignal?: AbortSignal): Promise<{ text: string; finalUrl: string; bytes: number }> {
const source = FRESHNESS_SOURCES.find(source => source.id === id)!;
const controller = new AbortController();
const signal = parentSignal ? AbortSignal.any([parentSignal, controller.signal]) : controller.signal;
const task = async () => {
let url: string = source.url;
for (let redirects = 0; ; redirects++) {
if (!allowedUrl(url)) throw new Error('redirect-not-allowlisted');
const response = await fetcher(url, { redirect: 'manual', signal, headers: { Accept: 'text/markdown, text/plain' } });
if (response.status >= 300 && response.status <= 399) {
await response.body?.cancel();
if (redirects >= FRESHNESS_BOUNDS.redirects) throw new Error('redirect-limit');
const location = response.headers.get('location');
if (!location) throw new Error('redirect-without-location');
url = new URL(location, url).href;
continue;
}
if (!response.ok) { await response.body?.cancel(); throw new Error(`http-${response.status}`); }
const data = await readBoundedResponse(response);
return { text: new TextDecoder('utf-8', { fatal: true }).decode(data), finalUrl: url, bytes: data.length };
}
};
return withinDeadline(task(), FRESHNESS_BOUNDS.responseMs, controller);
}
function section(text: string, title: string): string {
const tokens = marked.lexer(text);
if (tokens.filter(token => token.type === 'heading' && token.text === title).length !== 1) throw new Error(`missing-or-ambiguous-section:${title}`);
const start = tokens.findIndex(token => token.type === 'heading' && token.text === title);
if (start < 0) throw new Error(`missing-section:${title}`);
const heading = tokens[start] as { depth: number };
const end = tokens.findIndex((token, index) => index > start && token.type === 'heading' && token.depth <= heading.depth);
return tokens.slice(start + 1, end < 0 ? undefined : end).map(token => token.raw).join('');
}
function tables(text: string): string[][][] {
return marked.lexer(text).flatMap(token => {
if (token.type !== 'table') return [];
const table = token as Tokens.Table;
return [[table.header.map(cell => cell.text), ...table.rows.map(row => row.map(cell => cell.text))]];
});
}
function modelId(text: string): string {
const id = text.replace(/^`|`$/g, '').trim();
if (!/^[a-z][a-z0-9.-]{0,119}$/.test(id)) throw new Error('invalid-model-id');
return id;
}
export function lifecycleDate(value: string): string | null {
if (/^(N\/A|To be announced|-)$/i.test(value.trim())) return null;
const normalized = value.trim().replace(/\p{Dash_Punctuation}/gu, '-');
if (/^\d{4}-\d\d-\d\d$/.test(normalized)) {
const date = new Date(`${normalized}T00:00:00.000Z`);
if (!Number.isFinite(date.valueOf()) || date.toISOString().slice(0, 10) !== normalized) throw new Error('malformed-lifecycle-date');
return normalized;
}
const match = /^(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?) (\d{1,2}), (\d{4})$/.exec(value.trim());
if (!match) throw new Error('malformed-lifecycle-date');
const month = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(match[1].slice(0, 3));
const date = new Date(Date.UTC(Number(match[3]), month, Number(match[2])));
if (date.getUTCMonth() !== month || date.getUTCDate() !== Number(match[2])) throw new Error('malformed-lifecycle-date');
return date.toISOString().slice(0, 10);
}
function anthropicModels(text: string): ParsedSource {
const compare = section(text, 'Compare models');
const table = tables(compare).find(table => table[0][0] === 'Feature');
if (!table || table[0].length < 3) throw new Error('missing-comparison-table');
const ids = table.find(row => row[0] === 'Claude API ID')?.slice(1).map(modelId);
const pages = table.find(row => row[0] === 'Model page')?.slice(1);
if (!ids || !pages || ids.length !== pages.length) throw new Error('missing-api-id-row');
if (new Set(ids).size !== ids.length || !pages.every(page => /\]\(https:\/\/platform\.claude\.com\/docs\/en\/models\/[a-z0-9-]+\/overview\)/.test(page))) throw new Error('ambiguous-comparison-models');
const prose = marked.lexer(compare).filter(token => token.type === 'paragraph').map(token => token.raw).join('\n');
const smartLinks = [...prose.matchAll(/start with \[[^\]]+\]\(([^)]+)\) for most workloads/gi)];
const frontierLinks = [...prose.matchAll(/Use \[[^\]]+\]\(([^)]+)\) for demanding reasoning/gi)];
if (smartLinks.length !== 1 || frontierLinks.length !== 1) throw new Error('unrecognized-recommendation-structure');
const smart = smartLinks[0][1];
const frontier = frontierLinks[0][1];
const resolve = (url?: string): string => {
const index = pages.findIndex(page => /\]\(([^)]+)\)/.exec(page)?.[1] === url);
if (!url || index < 0) throw new Error('unrecognized-recommendation-structure');
return ids[index];
};
return { recommendations: { frontier: resolve(frontier), smart: resolve(smart) }, lineup: ids };
}
function openaiModels(text: string): ParsedSource {
const featured = section(text, 'Featured models');
const links = [...featured.matchAll(/^- \[[^\]]+\]\(\/api\/docs\/models\/([a-z0-9.-]+)\.md\): (.+)$/gm)];
const choose = (pattern: RegExp): string => {
const found = links.filter(link => pattern.test(link[2]));
if (found.length !== 1) throw new Error('unrecognized-recommendation-structure');
return modelId(found[0][1]);
};
const catalog = section(text, 'Browse our full catalog of models');
const lineup = [...catalog.matchAll(/^- \[[^\]]+\]\(\/api\/docs\/models\/([a-z0-9.-]+)\.md\): /gm)].map(match => modelId(match[1]));
if (!lineup.length) throw new Error('missing-model-catalog');
const recommendations = { frontier: choose(/^Start here for complex reasoning and coding\.$/i), smart: choose(/^Balance intelligence and cost\.$/i) };
if (!Object.values(recommendations).every(id => lineup.includes(id))) throw new Error('recommendation-not-in-catalog');
return { recommendations, lineup };
}
function anthropicLifecycle(text: string): ParsedSource {
const table = tables(section(text, 'Model status')).find(table => table[0][0] === 'API model name');
if (!table || table[0].join('|') !== 'API model name|Current state|Deprecated|Tentative retirement date' || table.length < 2) throw new Error('missing-lifecycle-table');
const lifecycle = table.slice(1).map(row => {
const states: Record<string, LifecycleRow['state']> = { Active: 'active', Legacy: 'legacy', Deprecated: 'deprecated', Retired: 'removed' };
const state = states[row[1]];
if (!state || row.length !== 4) throw new Error('unrecognized-lifecycle-state');
lifecycleDate(row[2]);
const floor = row[3].startsWith('Not sooner than ');
const date = lifecycleDate(row[3].replace(/^Not sooner than /, ''));
if (floor && (state === 'deprecated' || state === 'removed')) throw new Error('contradictory-lifecycle-floor');
return { modelId: modelId(row[0]), state, deadline: floor ? null : date, supportFloor: floor ? date : null };
});
return { lifecycle };
}
function openaiLifecycle(text: string): ParsedSource {
const upcoming = section(text, 'Upcoming deprecations');
const allTables = tables(text);
const modelTables = allTables.filter(table => table[0][0] === 'Shutdown date' && /^(?:Model (?:\/ system|family \/ snapshot|snapshot)|Deprecated model|Legacy model)$/.test(table[0][1]));
if (!upcoming || !modelTables.length) throw new Error('missing-deprecation-tables');
const lifecycle = modelTables.flatMap(table => table.slice(1).flatMap(row => {
if (/^(?:Videos API|Assistants API|OpenAI-Beta: |New fine-tuning training on )/.test(row[1]) || row[1].startsWith('`/')) return [];
if (!row[1].startsWith('`')) throw new Error('unrecognized-deprecated-model-cell');
const floor = row[0].startsWith('at earliest ');
const date = lifecycleDate(row[0].replace(/^at earliest /, ''));
if (!date) throw new Error('missing-shutdown-date');
const ids = [...row[1].matchAll(/`([a-z0-9.-]+)`/g)].map(match => modelId(match[1]));
if (!ids.length) throw new Error('unrecognized-deprecated-model-cell');
return ids.map(modelId => ({ modelId, state: 'deprecated' as const, deadline: floor ? null : date, supportFloor: floor ? date : null }));
}));
return { lifecycle };
}
export function parseOfficialSource(id: SourceId, text: string): ParsedSource {
if (new TextEncoder().encode(text).length > FRESHNESS_BOUNDS.responseBytes) throw new Error('response-too-large');
const parsers = { 'anthropic-models': anthropicModels, 'anthropic-lifecycle': anthropicLifecycle, 'openai-models': openaiModels, 'openai-lifecycle': openaiLifecycle };
return parsers[id](text);
}
export function parseOfficialSourceBounded(id: SourceId, text: string, signal: AbortSignal, createWorker: () => Worker = () => new Worker(new URL('./model-policy-freshness-parser-worker.ts', import.meta.url).href)): Promise<ParsedSource> {
if (signal.aborted) return Promise.reject(new Error('deadline-exceeded'));
return new Promise((resolve, reject) => {
const worker = createWorker();
const cleanup = () => { signal.removeEventListener('abort', abort); worker.terminate(); };
const abort = () => { cleanup(); reject(new Error('deadline-exceeded')); };
signal.addEventListener('abort', abort, { once: true });
worker.onmessage = (event: MessageEvent<{ ok: boolean; parsed: ParsedSource; error: string }>) => {
cleanup();
if (event.data.ok) resolve(event.data.parsed); else reject(new Error(event.data.error));
};
worker.onerror = () => { cleanup(); reject(new Error('parser-worker-unavailable')); };
try { worker.postMessage({ id, text }); } catch { cleanup(); reject(new Error('parser-worker-unavailable')); }
});
}
export async function collectFreshnessSources(fetcher: Fetcher = fetch, createWorker?: () => Worker): Promise<SourceResult[]> {
const controller = new AbortController();
const completed: Partial<Record<SourceId, SourceResult>> = {};
const tasks = Promise.all(FRESHNESS_SOURCES.map(async source => {
let result: SourceResult;
try {
const fetched = await fetchOfficialSource(source.id, fetcher, controller.signal);
const parsed = await parseOfficialSourceBounded(source.id, fetched.text, controller.signal, createWorker);
result = { id: source.id, receipt: { id: source.id, url: source.url, finalUrl: fetched.finalUrl, sha256: sha256(fetched.text), bytes: fetched.bytes, parsed } };
} catch (error) {
const reason = error instanceof Error ? error.message : 'source-unavailable';
result = { id: source.id, error: /^[a-z0-9: /.-]{1,160}$/i.test(reason) ? reason : 'source-unavailable' };
}
completed[source.id] = result;
return result;
}));
try { return await withinDeadline(tasks, FRESHNESS_BOUNDS.checkMs, controller); }
catch { return FRESHNESS_SOURCES.map(source => completed[source.id] ?? { id: source.id, error: 'complete-check-deadline-exceeded' }); }
}
+180
View File
@@ -0,0 +1,180 @@
import { FRESHNESS_SOURCES, type SourceResult, type SourceReceipt } from './model-policy-freshness-sources';
import { MODEL_CATALOG, MODEL_CATALOG_VERSION, modelCatalogSha256 } from './model-catalog';
import { sha256 } from './model-policy-freshness-sources';
export const FRESHNESS_SCHEMA_VERSION = 1;
export const FRESHNESS_PARSER_VERSION = 1;
export type FreshnessModel = { provider: 'anthropic' | 'openai'; tier: 'frontier' | 'smart'; modelId: string };
export type CatalogIdentity = { sha256: string; sourceSha256: string; version: number; models: FreshnessModel[] };
export type FreshnessRun = { id: string; attempt: number; startedAt: string; headSha: string };
export type FreshnessObservation = { catalog: CatalogIdentity; checkedAt: string; run: FreshnessRun; sources: SourceResult[] };
export type FreshnessStatus = 'current' | 'update-candidate' | 'stale' | 'unknown/source-unavailable';
export type LifecycleFinding = { key: string; provider: FreshnessModel['provider']; modelId: string; state: 'legacy' | 'deprecated' | 'removed'; deadline: string | null; firstObservedAt: string; lastObservedAt: string; sourceUrl: string; resolvedByCatalog?: string };
export type FreshnessEvidence = {
schemaVersion: 1; parserVersion: number; catalog: CatalogIdentity; checkedAt: string; nextCheckDueBy: string; run: FreshnessRun;
sources: Omit<SourceReceipt, 'parsed'>[];
};
export type FreshnessState = {
schemaVersion: 1; latest: { checkedAt: string; run: FreshnessRun }; lastSuccess: FreshnessEvidence | null;
lifecycle: LifecycleFinding[]; recoveryRequired: boolean;
};
export type FreshnessReport = {
schemaVersion: 1; status: FreshnessStatus; evidenceStatus: FreshnessStatus; complete: boolean;
errors: string[]; candidates: { provider: string; tier: string; current: string; proposed: string; reason: string }[];
state: FreshnessState;
};
const DAY = 86_400_000;
const iso = (value: string): boolean => /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(value) && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value;
const hash = (value: unknown): value is string => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
const string = (value: unknown): value is string => typeof value === 'string';
export const isFreshnessDate = (value: unknown): value is string => string(value) && iso(value);
export function freshnessCatalogIdentity(sourceText: string): CatalogIdentity {
return { sha256: modelCatalogSha256(), sourceSha256: sha256(sourceText), version: MODEL_CATALOG_VERSION, models: MODEL_CATALOG.map(entry => ({ provider: entry.provider, tier: entry.tier, modelId: entry.model })) };
}
export function validCatalog(value: unknown): value is CatalogIdentity {
if (!object(value) || !hash(value.sha256) || !hash(value.sourceSha256) || !Number.isInteger(value.version) || Number(value.version) < 1 || !Array.isArray(value.models) || value.models.length !== 4) return false;
const keys = new Set<string>();
for (const entry of value.models) {
if (!object(entry) || !['anthropic', 'openai'].includes(String(entry.provider)) || !['frontier', 'smart'].includes(String(entry.tier)) || !string(entry.modelId) || !/^[a-z][a-z0-9.-]{0,119}$/.test(entry.modelId)) return false;
keys.add(`${entry.provider}/${entry.tier}`);
}
return keys.size === 4;
}
export function validRun(value: unknown): value is FreshnessRun {
return object(value) && string(value.id) && /^\d{1,24}$/.test(value.id) && Number.isSafeInteger(value.attempt) && Number(value.attempt) > 0 && isFreshnessDate(value.startedAt) && string(value.headSha) && /^[a-f0-9]{40,64}$/.test(value.headSha);
}
function validReceipt(value: unknown, source: typeof FRESHNESS_SOURCES[number]): boolean {
return object(value) && value.id === source.id && value.url === source.url && hash(value.sha256)
&& (FRESHNESS_SOURCES.some(item => item.url === value.finalUrl) || value.finalUrl === 'https://developers.openai.com/api/docs/deprecations')
&& Number.isInteger(value.bytes) && Number(value.bytes) > 0 && Number(value.bytes) <= 2 * 1024 * 1024;
}
export function sameCatalog(a: CatalogIdentity, b: CatalogIdentity): boolean {
const keys = (catalog: CatalogIdentity) => catalog.models.map(model => `${model.provider}/${model.tier}/${model.modelId}`).sort().join('|');
return a.sha256 === b.sha256 && a.sourceSha256 === b.sourceSha256 && a.version === b.version && keys(a) === keys(b);
}
export function validLifecycle(value: unknown): value is LifecycleFinding[] {
return Array.isArray(value) && value.length <= 128 && new Set(value.map(item => object(item) ? item.key : undefined)).size === value.length && value.every(item => object(item)
&& ['anthropic', 'openai'].includes(String(item.provider)) && string(item.modelId) && /^[a-z][a-z0-9.-]{0,119}$/.test(item.modelId)
&& item.key === `${item.provider}/${item.modelId}` && ['legacy', 'deprecated', 'removed'].includes(String(item.state))
&& (item.deadline === null || (string(item.deadline) && /^\d{4}-\d\d-\d\d$/.test(item.deadline) && isFreshnessDate(`${item.deadline}T00:00:00.000Z`)))
&& isFreshnessDate(item.firstObservedAt) && isFreshnessDate(item.lastObservedAt)
&& FRESHNESS_SOURCES.some(source => source.id === `${item.provider}-lifecycle` && source.url === item.sourceUrl)
&& (item.resolvedByCatalog === undefined || hash(item.resolvedByCatalog)));
}
export function validEvidence(value: unknown): value is FreshnessEvidence {
if (!object(value) || value.schemaVersion !== 1 || value.parserVersion !== FRESHNESS_PARSER_VERSION || !validCatalog(value.catalog) || !isFreshnessDate(value.checkedAt) || !isFreshnessDate(value.nextCheckDueBy) || !validRun(value.run) || !Array.isArray(value.sources) || value.sources.length !== 4) return false;
if (Date.parse(value.nextCheckDueBy) !== Date.parse(value.checkedAt) + 7 * DAY) return false;
const sources = value.sources;
return Date.parse(value.run.startedAt) <= Date.parse(value.checkedAt)
&& FRESHNESS_SOURCES.every(source => sources.filter(receipt => validReceipt(receipt, source)).length === 1);
}
export function validState(value: unknown): value is FreshnessState {
if (!object(value) || value.schemaVersion !== 1 || !object(value.latest) || !isFreshnessDate(value.latest.checkedAt) || !validRun(value.latest.run)
|| (value.lastSuccess !== null && !validEvidence(value.lastSuccess)) || !validLifecycle(value.lifecycle) || typeof value.recoveryRequired !== 'boolean') return false;
if (Date.parse(value.latest.run.startedAt) > Date.parse(value.latest.checkedAt)) return false;
if (value.lastSuccess && (Date.parse(value.lastSuccess.checkedAt) > Date.parse(value.latest.checkedAt) || BigInt(value.lastSuccess.run.id) > BigInt(value.latest.run.id))) return false;
const checkedAt = value.latest.checkedAt;
return value.lifecycle.every(finding => Date.parse(finding.firstObservedAt) <= Date.parse(finding.lastObservedAt) && Date.parse(finding.lastObservedAt) <= Date.parse(checkedAt));
}
export function storedEvidenceStatus(catalog: CatalogIdentity, evidence: FreshnessEvidence | null, now: string): FreshnessStatus {
if (!evidence || !validEvidence(evidence) || !sameCatalog(catalog, evidence.catalog) || !isFreshnessDate(now) || Date.parse(evidence.checkedAt) > Date.parse(now)) return 'unknown/source-unavailable';
return Date.parse(now) - Date.parse(evidence.checkedAt) > 30 * DAY ? 'stale' : 'current';
}
export function observationIsOlder(observation: FreshnessObservation, latest: FreshnessState['latest']): boolean {
const a = observation.run;
const b = latest.run;
return BigInt(a.id) < BigInt(b.id) || (a.id === b.id && a.attempt <= b.attempt)
|| Date.parse(observation.checkedAt) < Date.parse(latest.checkedAt)
|| Date.parse(a.startedAt) < Date.parse(b.startedAt);
}
function sourceErrors(observation: FreshnessObservation): string[] {
return FRESHNESS_SOURCES.flatMap(source => {
const results = observation.sources.filter(result => result.id === source.id);
if (results.length !== 1) return [`${source.id}:missing-or-duplicate-source`];
const result = results[0];
if (result.error !== undefined) return [`${source.id}:${result.error}`];
const receipt = result.receipt;
if (!validReceipt(receipt, source)) return [`${source.id}:invalid-receipt`];
if (source.id.endsWith('-models') && (!receipt.parsed.recommendations || !receipt.parsed.lineup?.length)) return [`${source.id}:missing-recommendations`];
if (source.id.endsWith('-lifecycle') && !receipt.parsed.lifecycle?.length) return [`${source.id}:missing-lifecycle`];
return [];
});
}
function family(id: string): string {
if (id.startsWith('claude-')) return id.split('-')[1];
return /^gpt-\d+(?:\.\d+)?-([a-z]+)/.exec(id)?.[1] ?? id;
}
export function assessFreshness(observation: FreshnessObservation, previous: FreshnessState | null = null, recoveredLifecycle: LifecycleFinding[] = [], recoveryRequired = false): FreshnessReport {
if (!validCatalog(observation.catalog) || !validRun(observation.run) || !isFreshnessDate(observation.checkedAt) || Date.parse(observation.checkedAt) < Date.parse(observation.run.startedAt)) throw new Error('invalid-observation-identity');
const lifecycle = new Map((previous?.lifecycle ?? recoveredLifecycle).map(finding => [finding.key, { ...finding }]));
for (const finding of lifecycle.values()) {
if (observation.catalog.models.some(model => model.provider === finding.provider && model.modelId === finding.modelId)) delete finding.resolvedByCatalog;
}
const errors = sourceErrors(observation);
const candidates: FreshnessReport['candidates'] = [];
for (const model of observation.catalog.models) {
const models = observation.sources.find(source => source.id === `${model.provider}-models`)?.receipt?.parsed;
const rows = observation.sources.find(source => source.id === `${model.provider}-lifecycle`)?.receipt?.parsed.lifecycle;
if (model.provider === 'anthropic' && rows && !rows.some(row => row.modelId === model.modelId)) errors.push(`anthropic-lifecycle:missing-default:${model.modelId}`);
const row = rows?.find(row => row.modelId === model.modelId);
if (row && row.state !== 'active') {
const key = `${model.provider}/${model.modelId}`;
const previousFinding = lifecycle.get(key);
const rank = { legacy: 1, deprecated: 2, removed: 3 };
const observedState = row.state === 'deprecated' && row.deadline && row.deadline <= observation.checkedAt.slice(0, 10) ? 'removed' : row.state;
const state = previousFinding && rank[previousFinding.state] > rank[observedState] ? previousFinding.state : observedState;
const deadline = previousFinding?.deadline && (!row.deadline || previousFinding.deadline < row.deadline) ? previousFinding.deadline : row.deadline;
lifecycle.set(key, { key, provider: model.provider, modelId: model.modelId, state, deadline, sourceUrl: FRESHNESS_SOURCES.find(source => source.id === `${model.provider}-lifecycle`)!.url, firstObservedAt: previousFinding?.firstObservedAt ?? observation.checkedAt, lastObservedAt: observation.checkedAt });
}
const proposed = models?.recommendations?.[model.tier];
if (proposed && proposed !== model.modelId) candidates.push({ provider: model.provider, tier: model.tier, current: model.modelId, proposed, reason: family(proposed) === family(model.modelId) ? 'changed-monitored-family-recommendation' : 'unknown-replacement-family-needs-triage' });
if (models?.lineup && !models.lineup.includes(model.modelId) && proposed === model.modelId) errors.push(`${model.provider}-models:inconsistent-default:${model.modelId}`);
}
const complete = errors.length === 0;
if (complete) for (const finding of lifecycle.values()) {
if (!observation.catalog.models.some(model => model.provider === finding.provider && model.modelId === finding.modelId)) finding.resolvedByCatalog = observation.catalog.sha256;
}
const recovery = recoveryRequired || previous?.recoveryRequired === true;
if (recovery) errors.push('authoritative-state-recovery-required:manual-triage');
const lastSuccess: FreshnessEvidence | null = complete && !recovery ? {
schemaVersion: 1, parserVersion: FRESHNESS_PARSER_VERSION, catalog: observation.catalog, checkedAt: observation.checkedAt,
nextCheckDueBy: new Date(Date.parse(observation.checkedAt) + 7 * DAY).toISOString(), run: observation.run,
sources: observation.sources.map(source => { const { parsed, ...receipt } = source.receipt!; return receipt; }),
} : previous?.lastSuccess ?? null;
const evidenceStatus = storedEvidenceStatus(observation.catalog, lastSuccess, observation.checkedAt);
const unresolved = [...lifecycle.values()].some(finding => !finding.resolvedByCatalog);
const status: FreshnessStatus = errors.length ? 'unknown/source-unavailable' : unresolved || candidates.length ? 'update-candidate' : evidenceStatus;
return { schemaVersion: 1, status, evidenceStatus, complete: complete && !recovery, errors, candidates, state: { schemaVersion: 1, latest: { checkedAt: observation.checkedAt, run: observation.run }, lastSuccess, lifecycle: [...lifecycle.values()].sort((a, b) => a.key.localeCompare(b.key)), recoveryRequired: recovery } };
}
export function renderFreshnessReport(report: FreshnessReport): string {
const evidence = report.state.lastSuccess;
const lines = [
`## Model-policy freshness: ${report.status}`, '',
`Checked observation: ${report.state.latest.checkedAt}; run ${report.state.latest.run.id}/${report.state.latest.run.attempt}.`,
`Last complete trustworthy check: ${evidence?.checkedAt ?? 'unavailable'}. Next scheduled check due by: ${evidence?.nextCheckDueBy ?? 'unavailable'}.`,
`Stored evidence status: ${report.evidenceStatus}; trustworthy evidence older than thirty days is stale, independently of the seven-day scheduled due date.`,
'The seven-day due date is a manual-check mitigation, not automatic liveness monitoring. If GitHub never runs this job, it cannot alert on its own absence.',
'Public API recommendations only; this does not certify account entitlement, quality superiority, CLI compatibility, or Bedrock/Vertex/custom endpoint availability.', '',
];
for (const error of report.errors) lines.push(`- Source/recovery finding: ${error}. Triage within seven days; manually check the named official source and the last successful workflow date.`);
for (const candidate of report.candidates) lines.push(`- Candidate (${candidate.provider}/${candidate.tier}): ${candidate.current} → ${candidate.proposed}; ${candidate.reason}. Review within thirty days; qualify a reviewed catalog PR or record a retain decision.`);
for (const finding of report.state.lifecycle.filter(finding => !finding.resolvedByCatalog)) lines.push(`- ${finding.state === 'legacy' ? 'Lifecycle triage' : 'URGENT'}: ${finding.modelId} is ${finding.state}; removal deadline ${finding.deadline ?? 'not announced'}. Source: ${finding.sourceUrl}. ${finding.state === 'legacy' ? 'Review within thirty days; legacy does not announce retirement.' : 'Triage within one working day and qualify or explicitly mitigate before removal; already-removed defaults remain unresolved.'}`);
lines.push('', 'No catalog updates, pin rewrites, model switches, or paid evaluations are performed. Candidate qualification requires separate budget approval and pinned baseline/candidate inputs.', 'Manual recovery: docs/troubleshooting.md#model-policy-freshness. Preserve lifecycle history and dispositions when repairing the owned state; unresolved lifecycle findings cannot be cleared by a failed fetch.', '', '### Official source receipts');
for (const receipt of evidence?.sources ?? []) lines.push(`- ${receipt.id}: ${receipt.url}; SHA-256 ${receipt.sha256}; ${receipt.bytes} bytes.`);
return lines.join('\n');
}
+44
View File
@@ -0,0 +1,44 @@
/** One-time plan-review policy notice, emitted only at an actual role-bearing invocation. */
import * as fs from 'node:fs';
import * as path from 'node:path';
import { MODEL_POLICY_VERSION, PROVIDER_ENV_OVERRIDES, describeSelection, modelPolicyCommands, type ModelSelection } from './model-policy';
import { resolveStateRoot, type StateRootEnv } from './state-root';
export function modelPolicyNoticeMarker(env: StateRootEnv = process.env, platform: NodeJS.Platform = process.platform): string {
return path.join(resolveStateRoot(env, platform), `.model-policy-notice-v${MODEL_POLICY_VERSION}`);
}
export function modelPolicyNoticeText(selection: ModelSelection): string {
const commands = modelPolicyCommands();
return [
'NOTICE: gstack plan reviews now use an independent plan-review model (default: frontier tier), not your general coding-model setting.',
`This review: ${describeSelection(selection)}.`,
`Explicit choices still win, in order: a model named for the request, ${PROVIDER_ENV_OVERRIDES[selection.provider]}, then the per-tier config override.`,
`Use the smart tier: ${commands.useSmart}. Keep host model settings: ${commands.useHost}. Shown once. Docs: https://github.com/garrytan/gstack/blob/main/docs/model-policy.md`,
].join('\n');
}
/**
* Show before the paid call; record the marker only afterwards. A marker that
* cannot be written repeats the notice next time instead of hiding it.
*/
export function emitModelPolicyNotice(selection: ModelSelection, opts: {
env?: StateRootEnv;
platform?: NodeJS.Platform;
write?: (text: string) => void;
} = {}): boolean {
let marker: string;
try {
marker = modelPolicyNoticeMarker(opts.env, opts.platform);
if (fs.existsSync(marker)) return false;
} catch {
marker = '';
}
(opts.write ?? ((text: string) => process.stderr.write(text)))(`${modelPolicyNoticeText(selection)}\n`);
if (!marker) return true;
try {
fs.mkdirSync(path.dirname(marker), { recursive: true });
fs.writeFileSync(marker, '');
} catch {}
return true;
}
+661
View File
@@ -0,0 +1,661 @@
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { catalogEntry, MODEL_CATALOG_VERSION, MODEL_PROVIDERS, type ModelCatalogEntry, type ModelProvider, type ModelTier } from './model-catalog';
import { resolveStateRoot, type StateRootEnv } from './state-root';
export const MODEL_POLICY_VERSION = 1 as const;
export type ModelRole = 'plan-review' | 'implementation';
export type PlanReviewTier = ModelTier | 'host';
export type ModelHost = 'claude-code' | 'codex' | 'unknown';
export const MODEL_ROLES: readonly ModelRole[] = Object.freeze(['plan-review', 'implementation'] as const);
export const PLAN_REVIEW_TIERS: readonly PlanReviewTier[] = Object.freeze(['frontier', 'smart', 'host'] as const);
export const IMPLEMENTATION_TIERS: readonly ModelTier[] = Object.freeze(['frontier', 'smart'] as const);
export type PolicyConfigKey =
| 'plan_review_tier'
| 'implementation_tier'
| 'model_frontier_claude'
| 'model_frontier_openai'
| 'model_smart_claude'
| 'model_smart_openai';
export const POLICY_CONFIG_KEYS: readonly PolicyConfigKey[] = Object.freeze([
'plan_review_tier',
'implementation_tier',
'model_frontier_claude',
'model_frontier_openai',
'model_smart_claude',
'model_smart_openai',
] as const);
export const MODEL_OVERRIDE_KEYS: Readonly<Record<ModelProvider, Readonly<Record<ModelTier, PolicyConfigKey>>>> = Object.freeze({
anthropic: Object.freeze({ frontier: 'model_frontier_claude', smart: 'model_smart_claude' } as const),
openai: Object.freeze({ frontier: 'model_frontier_openai', smart: 'model_smart_openai' } as const),
});
export const PROVIDER_ENV_OVERRIDES = Object.freeze({ anthropic: 'GSTACK_CLAUDE_MODEL', openai: 'GSTACK_CODEX_MODEL' } as const);
export const OPENAI_MODEL_ID_PATTERN = /^[A-Za-z0-9._:/-]{1,100}$/;
export const ANTHROPIC_MODEL_ID_PATTERN = /^[A-Za-z0-9._:/@+[\]-]{1,512}$/;
const DOCS = 'https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md';
const DOCS_CONFIG = `${DOCS}#model-policy-config`;
const DOCS_PROVIDER = `${DOCS}#model-policy-provider`;
const DOCS_SELECTION = `${DOCS}#model-policy-selection`;
const PROVIDER_LABEL: Record<ModelProvider, string> = { anthropic: 'Anthropic (Claude Code)', openai: 'OpenAI (Codex)' };
export type ModelPolicyReason =
| 'config_unreadable'
| 'config_malformed'
| 'empty_value'
| 'invalid_plan_review_tier'
| 'invalid_implementation_tier'
| 'invalid_model_id'
| 'custom_provider_requires_model'
| 'native_provider_unresolved'
| 'provider_required';
export type SelectionSource =
| { kind: 'request'; label: string }
| { kind: 'env'; name: (typeof PROVIDER_ENV_OVERRIDES)[ModelProvider]; label: string }
| { kind: 'config'; key: PolicyConfigKey; path: string; label: string }
| { kind: 'catalog'; catalogVersion: typeof MODEL_CATALOG_VERSION; verifiedAt: string; sourceUrl: string; label: string }
| { kind: 'host'; reason: 'host-mode'; key: 'plan_review_tier'; label: string };
export interface ModelPolicyErrorJson {
reason: ModelPolicyReason;
problem: string;
cause: string;
key: string | null;
source: SelectionSource['kind'] | 'native' | null;
repair: string[];
docs: string;
}
export class ModelPolicyError extends Error {
readonly reason: ModelPolicyReason;
readonly problem: string;
readonly cause: string;
readonly key: string | null;
readonly source: ModelPolicyErrorJson['source'];
readonly repair: string[];
readonly docs: string;
constructor(detail: ModelPolicyErrorJson) {
super(`${detail.problem}: ${detail.cause}`);
this.name = 'ModelPolicyError';
this.reason = detail.reason;
this.problem = detail.problem;
this.cause = detail.cause;
this.key = detail.key;
this.source = detail.source;
this.repair = detail.repair;
this.docs = detail.docs;
}
toJSON(): ModelPolicyErrorJson {
return { reason: this.reason, problem: this.problem, cause: this.cause, key: this.key, source: this.source, repair: this.repair, docs: this.docs };
}
}
export interface TierChoice<T extends PlanReviewTier> {
value: T;
key: 'plan_review_tier' | 'implementation_tier';
origin: 'config' | 'default';
path: string;
}
export interface ModelPolicyConfig {
path: string;
exists: boolean;
planReviewTier: TierChoice<PlanReviewTier>;
implementationTier: TierChoice<ModelTier>;
overrides: Record<ModelProvider, Record<ModelTier, string | null>>;
}
interface SelectionBase {
role: ModelRole;
provider: ModelProvider;
tier: PlanReviewTier;
tierFrom: TierChoice<PlanReviewTier>;
catalog: ModelCatalogEntry | null;
}
export interface SelectedModel extends SelectionBase {
status: 'selected';
requestedModel: string;
source: Exclude<SelectionSource, { kind: 'host' }>;
}
export interface DelegatedHostModel extends SelectionBase {
status: 'delegated-host';
requestedModel: null;
source: Extract<SelectionSource, { kind: 'host' }>;
}
export type ModelSelection = SelectedModel | DelegatedHostModel;
function sanitize(value: string): string {
return value.replace(/[\x00-\x1f\x7f]/g, ' ').slice(0, 200);
}
function providerOfKey(key: PolicyConfigKey): ModelProvider | null {
for (const provider of MODEL_PROVIDERS) {
if (Object.values(MODEL_OVERRIDE_KEYS[provider]).includes(key)) return provider;
}
return null;
}
export function isValidModelId(provider: ModelProvider, value: string): boolean {
return (provider === 'openai' ? OPENAI_MODEL_ID_PATTERN : ANTHROPIC_MODEL_ID_PATTERN).test(value);
}
function modelIdRule(provider: ModelProvider): string {
return provider === 'openai'
? '1-100 characters from A-Z a-z 0-9 . _ : / -'
: '1-512 characters from A-Z a-z 0-9 . _ : / @ + [ ] -';
}
export function validatePolicyValue(key: PolicyConfigKey, rawValue: string, configPath = 'config.yaml'): ModelPolicyError | null {
const value = rawValue.trim();
if (value === '') {
return new ModelPolicyError({
reason: 'empty_value',
problem: `${key} is empty in ${configPath}`,
cause: 'an empty policy value is not a choice; unset the key to use the default',
key,
source: 'config',
repair: [`gstack-config unset ${key}`],
docs: DOCS_CONFIG,
});
}
if (key === 'plan_review_tier' && !(PLAN_REVIEW_TIERS as readonly string[]).includes(value)) {
return new ModelPolicyError({
reason: 'invalid_plan_review_tier',
problem: `plan_review_tier '${sanitize(value)}' in ${configPath} is not recognized`,
cause: 'valid values are frontier, smart and host',
key,
source: 'config',
repair: ['gstack-config unset plan_review_tier'],
docs: DOCS_CONFIG,
});
}
if (key === 'implementation_tier' && !(IMPLEMENTATION_TIERS as readonly string[]).includes(value)) {
return new ModelPolicyError({
reason: 'invalid_implementation_tier',
problem: `implementation_tier '${sanitize(value)}' in ${configPath} is not recognized`,
cause: 'valid values are frontier and smart',
key,
source: 'config',
repair: ['gstack-config unset implementation_tier'],
docs: DOCS_CONFIG,
});
}
const provider = providerOfKey(key);
if (provider && !isValidModelId(provider, value)) {
return new ModelPolicyError({
reason: 'invalid_model_id',
problem: `${key} '${sanitize(value)}' in ${configPath} is not a valid ${provider} model ID`,
cause: `use ${modelIdRule(provider)}`,
key,
source: 'config',
repair: [`gstack-config set ${key} <model-id>`, `gstack-config unset ${key}`],
docs: DOCS_CONFIG,
});
}
return null;
}
export function parseModelPolicyConfig(text: string, configPath: string): ModelPolicyConfig {
const records = new Map<PolicyConfigKey, string>();
for (const line of text.split('\n')) {
for (const key of POLICY_CONFIG_KEYS) {
if (line.startsWith(`${key}:`)) {
records.set(key, line.slice(key.length + 1).trim());
} else if (new RegExp(`^\\s*${key}\\s*:`).test(line)) {
throw new ModelPolicyError({
reason: 'config_malformed',
problem: `${configPath} has a malformed ${key} record`,
cause: `write it as '${key}: <value>' at the start of the line, with no indentation or space before the colon`,
key,
source: 'config',
repair: [`gstack-config unset ${key}`, `gstack-config set ${key} <value>`],
docs: DOCS_CONFIG,
});
}
}
}
const configValue = (key: PolicyConfigKey): string | null => {
const value = records.get(key);
if (value === undefined) return null;
const problem = validatePolicyValue(key, value, configPath);
if (problem) throw problem;
return value;
};
const planReviewTier = configValue('plan_review_tier') as PlanReviewTier | null;
const implementationTier = configValue('implementation_tier') as ModelTier | null;
return {
path: configPath,
exists: true,
planReviewTier: { value: planReviewTier ?? 'frontier', key: 'plan_review_tier', origin: planReviewTier ? 'config' : 'default', path: configPath },
implementationTier: { value: implementationTier ?? 'smart', key: 'implementation_tier', origin: implementationTier ? 'config' : 'default', path: configPath },
overrides: {
anthropic: { frontier: configValue('model_frontier_claude'), smart: configValue('model_smart_claude') },
openai: { frontier: configValue('model_frontier_openai'), smart: configValue('model_smart_openai') },
},
};
}
export function readModelPolicyConfig(opts: { env?: StateRootEnv; platform?: NodeJS.Platform } = {}): ModelPolicyConfig {
const configPath = `${resolveStateRoot(opts.env ?? process.env, opts.platform ?? process.platform)}/config.yaml`;
let text: string;
try {
text = fs.readFileSync(configPath, 'utf-8');
} catch (error) {
const code = (error as NodeJS.ErrnoException).code ?? 'unknown error';
if (code === 'ENOENT') return { ...parseModelPolicyConfig('', configPath), exists: false };
const cause = code === 'EISDIR' ? 'config.yaml is a directory'
: code === 'EACCES' || code === 'EPERM' ? 'permission denied'
: code === 'ENOTDIR' ? 'the state root is not a directory'
: `the read failed (${code})`;
throw new ModelPolicyError({
reason: 'config_unreadable',
problem: `gstack cannot read ${configPath}, so the model policy is unknown and no outside model call will start`,
cause,
key: null,
source: 'config',
repair: [`fix the permissions or type of ${configPath}`, 'or point GSTACK_HOME at a readable state root'],
docs: DOCS_CONFIG,
});
}
return parseModelPolicyConfig(text, configPath);
}
const TRUTHY = new Set(['1', 'true', 'yes', 'on']);
function truthy(value: unknown): boolean {
return typeof value === 'string' && TRUTHY.has(value.trim().toLowerCase());
}
function nonDefaultUrl(value: unknown, defaultUrl: string): boolean {
return typeof value === 'string' && value.trim() !== '' && value.trim().replace(/\/+$/, '') !== defaultUrl;
}
function homeDir(env: StateRootEnv, platform: NodeJS.Platform): string {
return env.HOME || (platform === 'win32' ? env.USERPROFILE || '' : '') || os.homedir();
}
function unresolved(provider: ModelProvider, key: string, cause: string, repair: string): ModelPolicyError {
return new ModelPolicyError({
reason: 'native_provider_unresolved',
problem: `gstack cannot tell which ${PROVIDER_LABEL[provider]} endpoint your native settings use`,
cause,
key,
source: 'native',
repair: [repair, 'or name the model explicitly for this request'],
docs: DOCS_PROVIDER,
});
}
const CLAUDE_FLAGS = ['CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY'] as const;
export interface NativeSettingsOptions {
env?: StateRootEnv;
platform?: NodeJS.Platform;
cwd?: string;
claudeManagedDir?: string;
codexSystemConfig?: string | null;
}
export interface CustomProviderDetection {
custom: boolean;
signals: string[];
scanned: string[];
unscanned: string[];
}
export const CUSTOM_PROVIDER_UNSCANNED: Readonly<Record<ModelProvider, readonly string[]>> = Object.freeze({
anthropic: Object.freeze(['server-managed settings from the claude.ai console', 'MDM or OS policy (macOS profile, Windows registry)', 'a --settings flag passed to claude']),
openai: Object.freeze(['cloud-managed Codex config', '-c, --config or --profile flags passed to codex']),
});
function claudeManagedDir(platform: NodeJS.Platform): string {
if (platform === 'darwin') return '/Library/Application Support/ClaudeCode';
if (platform === 'win32') return 'C:\\Program Files\\ClaudeCode';
return '/etc/claude-code';
}
function gitRoots(cwd: string): string[] {
for (let dir = cwd; ; dir = path.dirname(dir)) {
const dotGit = path.join(dir, '.git');
let stat: fs.Stats | null = null;
try {
stat = fs.lstatSync(dotGit);
} catch {
stat = null;
}
if (stat?.isDirectory()) return [dir];
if (stat?.isFile()) {
try {
const gitDir = path.resolve(dir, fs.readFileSync(dotGit, 'utf-8').replace(/^gitdir:\s*/, '').trim());
const commonDir = path.resolve(gitDir, fs.readFileSync(path.join(gitDir, 'commondir'), 'utf-8').trim());
return path.basename(commonDir) === '.git' ? [dir, path.dirname(commonDir)] : [dir];
} catch {
return [dir];
}
}
if (path.dirname(dir) === dir) return [];
}
}
function claudeSettingsFiles(env: StateRootEnv, platform: NodeJS.Platform, cwd: string, managedDir: string): string[] {
const files = [
path.join(env.CLAUDE_CONFIG_DIR || path.join(homeDir(env, platform), '.claude'), 'settings.json'),
path.join(cwd, '.claude', 'settings.json'),
path.join(cwd, '.claude', 'settings.local.json'),
...gitRoots(cwd).map(root => path.join(root, '.claude', 'settings.local.json')),
path.join(managedDir, 'managed-settings.json'),
];
const dropIns = path.join(managedDir, 'managed-settings.d');
try {
files.push(...fs.readdirSync(dropIns).filter(name => name.endsWith('.json') && !name.startsWith('.')).sort().map(name => path.join(dropIns, name)));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw unresolved('anthropic', dropIns, `${dropIns} could not be read`, `fix the permissions of ${dropIns}`);
}
}
return [...new Set(files)];
}
function claudeSettingsEnv(file: string): Record<string, unknown> | null {
let raw: string;
try {
raw = fs.readFileSync(file, 'utf-8');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw unresolved('anthropic', file, `${file} could not be read`, `fix the permissions of ${file}`);
}
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
throw unresolved('anthropic', file, `${file} is not valid JSON`, `fix the JSON in ${file}`);
}
const block = (parsed as { env?: unknown } | null)?.env;
if (block === undefined) return {};
if (typeof block !== 'object' || block === null || Array.isArray(block)) {
throw unresolved('anthropic', file, `the env block in ${file} is not an object`, `fix the env block in ${file}`);
}
for (const name of [...CLAUDE_FLAGS, 'ANTHROPIC_BASE_URL']) {
const value = (block as Record<string, unknown>)[name];
if (value !== undefined && typeof value !== 'string') {
throw unresolved('anthropic', `${file} env.${name}`, `env.${name} in ${file} is not a string`, `fix env.${name} in ${file}`);
}
}
return block as Record<string, unknown>;
}
function claudeSignals(env: StateRootEnv, platform: NodeJS.Platform, cwd: string, managedDir: string): { signals: string[]; scanned: string[] } {
const sources: Array<[string, Record<string, unknown>]> = [['environment', env]];
const scanned = ['environment'];
for (const file of claudeSettingsFiles(env, platform, cwd, managedDir)) {
scanned.push(file);
const block = claudeSettingsEnv(file);
if (block) sources.push([file, block]);
}
const signals: string[] = [];
for (const [origin, values] of sources) {
for (const flag of CLAUDE_FLAGS) {
if (truthy(values[flag])) signals.push(`${flag} (${origin})`);
}
if (nonDefaultUrl(values.ANTHROPIC_BASE_URL, 'https://api.anthropic.com')) signals.push(`ANTHROPIC_BASE_URL (${origin})`);
}
return { signals, scanned };
}
function readToml(file: string): Record<string, unknown> | null {
let raw: string;
try {
raw = fs.readFileSync(file, 'utf-8');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw unresolved('openai', file, `${file} could not be read`, `fix the permissions of ${file}`);
}
try {
return Bun.TOML.parse(raw) as Record<string, unknown>;
} catch {
throw unresolved('openai', file, `${file} is not valid TOML`, `fix the TOML in ${file}`);
}
}
function codexSignals(env: StateRootEnv, platform: NodeJS.Platform, systemConfig: string | null): { signals: string[]; scanned: string[] } {
const codexHome = env.CODEX_HOME || path.join(homeDir(env, platform), '.codex');
const signals: string[] = [];
if (nonDefaultUrl(env.OPENAI_BASE_URL, 'https://api.openai.com/v1')) signals.push('OPENAI_BASE_URL (environment)');
if (!path.isAbsolute(codexHome)) {
throw unresolved('openai', 'CODEX_HOME', `CODEX_HOME '${sanitize(codexHome)}' is not an absolute path`, 'set CODEX_HOME to an absolute path');
}
const configPath = path.join(codexHome, 'config.toml');
const scanned = ['environment', configPath, ...(systemConfig ? [systemConfig] : [])];
const user = readToml(configPath) ?? {};
const layers: Array<[string, Record<string, unknown>]> = [];
if (user.profile !== undefined) {
const profiles = user.profiles as Record<string, unknown> | undefined;
const selected = typeof user.profile === 'string' && typeof profiles === 'object' && profiles !== null ? profiles[user.profile] : undefined;
if (typeof selected !== 'object' || selected === null || Array.isArray(selected)) {
throw unresolved('openai', `${configPath} profile`, `the active profile in ${configPath} is not a defined [profiles.<name>] table`, `define the profile or remove profile from ${configPath}`);
}
layers.push([`${configPath} [profiles.${sanitize(user.profile as string)}]`, selected as Record<string, unknown>]);
}
layers.push([configPath, user]);
if (systemConfig) layers.push([systemConfig, readToml(systemConfig) ?? {}]);
for (const key of ['model_provider', 'openai_base_url']) {
const winner = layers.find(([, layer]) => layer[key] !== undefined);
if (!winner) continue;
const [origin, layer] = winner;
const value = layer[key];
if (typeof value !== 'string') {
throw unresolved('openai', `${origin} ${key}`, `${key} in ${origin} is not a string`, `fix ${key} in ${origin}`);
}
if (key === 'model_provider' && value !== 'openai') signals.push(`model_provider "${sanitize(value)}" (${origin})`);
if (key === 'openai_base_url' && nonDefaultUrl(value, 'https://api.openai.com/v1')) signals.push(`openai_base_url (${origin})`);
}
return { signals, scanned };
}
export function detectCustomProvider(provider: ModelProvider, opts: NativeSettingsOptions = {}): CustomProviderDetection {
const env = opts.env ?? process.env;
const platform = opts.platform ?? process.platform;
const { signals, scanned } = provider === 'anthropic'
? claudeSignals(env, platform, path.resolve(opts.cwd ?? process.cwd()), opts.claudeManagedDir ?? claudeManagedDir(platform))
: codexSignals(env, platform, opts.codexSystemConfig !== undefined ? opts.codexSystemConfig : platform === 'win32' ? null : '/etc/codex/config.toml');
return { custom: signals.length > 0, signals, scanned, unscanned: [...CUSTOM_PROVIDER_UNSCANNED[provider]] };
}
function sourceLabel(provider: ModelProvider, tier: ModelTier, entry: ModelCatalogEntry): string {
return `gstack catalog ${tier}/${provider} (verified ${entry.verifiedAt})`;
}
function checkedSource(provider: ModelProvider, raw: string, source: SelectedModel['source']): { model: string; source: SelectedModel['source'] } {
const model = raw.trim();
if (isValidModelId(provider, model)) return { model, source };
const repair = source.kind === 'env'
? [`unset ${source.name}`, `export ${source.name}=<model-id>`]
: ['name a valid model ID for this request, or omit it to use the configured policy'];
throw new ModelPolicyError({
reason: 'invalid_model_id',
problem: `the ${provider} model '${sanitize(raw)}' from ${source.label} is not a valid model ID`,
cause: `use ${modelIdRule(provider)}; ${source.label} outranks gstack's tier settings, so changing a tier will not help`,
key: source.kind === 'env' ? source.name : 'request',
source: source.kind,
repair,
docs: DOCS_SELECTION,
});
}
function hostTool(provider: ModelProvider): string {
return provider === 'anthropic' ? 'Claude Code' : 'Codex';
}
export function resolvePlanReviewModel(opts: {
provider: ModelProvider;
requestedModel?: string;
env?: StateRootEnv;
platform?: NodeJS.Platform;
cwd?: string;
claudeManagedDir?: string;
codexSystemConfig?: string | null;
config?: ModelPolicyConfig;
}): ModelSelection {
const env = opts.env ?? process.env;
const platform = opts.platform ?? process.platform;
const config = opts.config ?? readModelPolicyConfig({ env, platform });
const { provider } = opts;
const tierFrom = config.planReviewTier;
const tier = tierFrom.value;
const base = { role: 'plan-review' as const, provider, tier, tierFrom, catalog: tier === 'host' ? null : catalogEntry(provider, tier) };
if (opts.requestedModel !== undefined) {
const { model, source } = checkedSource(provider, opts.requestedModel, { kind: 'request', label: 'explicit request' });
return { ...base, status: 'selected', requestedModel: model, source };
}
const envName = PROVIDER_ENV_OVERRIDES[provider];
const envValue = env[envName];
if (envValue) {
const { model, source } = checkedSource(provider, envValue, { kind: 'env', name: envName, label: `${envName} environment variable` });
return { ...base, status: 'selected', requestedModel: model, source };
}
if (tier === 'host') {
return {
...base,
status: 'delegated-host',
requestedModel: null,
source: { kind: 'host', reason: 'host-mode', key: 'plan_review_tier', label: `plan_review_tier=host: ${hostTool(provider)}'s own settings choose the model` },
};
}
const overrideKey = MODEL_OVERRIDE_KEYS[provider][tier];
const override = config.overrides[provider][tier];
if (override) {
return { ...base, status: 'selected', requestedModel: override, source: { kind: 'config', key: overrideKey, path: config.path, label: `${overrideKey} in ${config.path}` } };
}
const entry = catalogEntry(provider, tier);
const custom = detectCustomProvider(provider, { ...opts, env, platform });
if (custom.custom) {
throw new ModelPolicyError({
reason: 'custom_provider_requires_model',
problem: `no ${tier} plan-review model is set for your custom ${PROVIDER_LABEL[provider]} endpoint`,
cause: `custom routing is active (${custom.signals.join(', ')}), so gstack will not send the public catalog ID ${entry.model} to it`,
key: custom.signals[0] ?? null,
source: 'native',
repair: [`gstack-config set ${overrideKey} <model-id-your-endpoint-serves>`, `export ${envName}=<model-id-your-endpoint-serves>`, 'gstack-config set plan_review_tier host'],
docs: DOCS_PROVIDER,
});
}
return {
...base,
status: 'selected',
requestedModel: entry.model,
source: { kind: 'catalog', catalogVersion: MODEL_CATALOG_VERSION, verifiedAt: entry.verifiedAt, sourceUrl: entry.sourceUrl, label: sourceLabel(provider, tier, entry) },
};
}
export function resolveImplementationModels(opts: {
provider?: ModelProvider;
host?: ModelHost;
requestedModel?: string;
env?: StateRootEnv;
platform?: NodeJS.Platform;
cwd?: string;
claudeManagedDir?: string;
codexSystemConfig?: string | null;
config?: ModelPolicyConfig;
} = {}): ModelSelection[] {
const env = opts.env ?? process.env;
const platform = opts.platform ?? process.platform;
const providers: ModelProvider[] = opts.provider ? [opts.provider]
: opts.host === 'claude-code' ? ['anthropic']
: opts.host === 'codex' ? ['openai']
: [...MODEL_PROVIDERS];
if (opts.requestedModel !== undefined && providers.length !== 1) {
throw new ModelPolicyError({
reason: 'provider_required',
problem: 'an explicit implementation model needs a provider',
cause: 'the host does not identify one provider, and gstack does not infer a provider from a model name',
key: 'request',
source: 'request',
repair: ['pass the provider (anthropic or openai) with the requested model'],
docs: DOCS_SELECTION,
});
}
const config = opts.config ?? readModelPolicyConfig({ env, platform });
const tierFrom = config.implementationTier;
const tier = tierFrom.value;
return providers.map((provider): ModelSelection => {
const entry = catalogEntry(provider, tier);
const base = { role: 'implementation' as const, provider, tier, tierFrom, catalog: entry };
if (opts.requestedModel !== undefined) {
const { model, source } = checkedSource(provider, opts.requestedModel, { kind: 'request', label: 'explicit request' });
return { ...base, status: 'selected', requestedModel: model, source };
}
const overrideKey = MODEL_OVERRIDE_KEYS[provider][tier];
const override = config.overrides[provider][tier];
if (override) {
return { ...base, status: 'selected', requestedModel: override, source: { kind: 'config', key: overrideKey, path: config.path, label: `${overrideKey} in ${config.path}` } };
}
const custom = detectCustomProvider(provider, { ...opts, env, platform });
if (custom.custom) {
throw new ModelPolicyError({
reason: 'custom_provider_requires_model',
problem: `no ${tier} implementation model is set for your custom ${PROVIDER_LABEL[provider]} endpoint`,
cause: `custom routing is active (${custom.signals.join(', ')}), so gstack will not recommend the public catalog ID ${entry.model} for it`,
key: custom.signals[0] ?? null,
source: 'native',
repair: [`gstack-config set ${overrideKey} <model-id-your-endpoint-serves>`, 'or name the implementation model explicitly for this request'],
docs: DOCS_PROVIDER,
});
}
return {
...base,
status: 'selected',
requestedModel: entry.model,
source: { kind: 'catalog', catalogVersion: MODEL_CATALOG_VERSION, verifiedAt: entry.verifiedAt, sourceUrl: entry.sourceUrl, label: sourceLabel(provider, tier, entry) },
};
});
}
export function describeSelection(selection: ModelSelection): string {
const model = selection.status === 'selected' ? selection.requestedModel : 'host-controlled (unknown until it runs)';
return `${selection.role} via ${selection.provider}: ${model} [tier ${selection.tier}; from ${selection.source.label}]`;
}
export function selectionRepair(selection: ModelSelection): string[] {
const { source, provider } = selection;
if (source.kind === 'request') return ['name a different model for this request, or omit it to use the configured policy'];
if (source.kind === 'env') return [`unset ${source.name}`, `export ${source.name}=<model-id>`];
if (source.kind === 'config') return [`gstack-config set ${source.key} <model-id>`, `gstack-config unset ${source.key}`];
const tierKey = selection.role === 'plan-review' ? 'plan_review_tier' : 'implementation_tier';
if (source.kind === 'host') {
const native = provider === 'anthropic' ? 'choose the model in Claude Code settings' : 'set model in ${CODEX_HOME:-~/.codex}/config.toml';
return [native, `export ${PROVIDER_ENV_OVERRIDES[provider]}=<model-id>`, 'gstack-config unset plan_review_tier'];
}
const tier = selection.tier as ModelTier;
const other = tier === 'frontier' ? 'smart' : 'frontier';
return [
`gstack-config set ${MODEL_OVERRIDE_KEYS[provider][tier]} <model-id>`,
`gstack-config set ${tierKey} ${other}`,
...(selection.role === 'plan-review' ? ['gstack-config set plan_review_tier host'] : []),
];
}
export function modelPolicyCommands() {
return {
useSmart: 'gstack-config set plan_review_tier smart',
useHost: 'gstack-config set plan_review_tier host',
restoreDefault: 'gstack-config unset plan_review_tier',
overrideExample: (provider: ModelProvider, tier: ModelTier) => `gstack-config set ${MODEL_OVERRIDE_KEYS[provider][tier]} <model-id>`,
unsetOverride: (provider: ModelProvider, tier: ModelTier) => `gstack-config unset ${MODEL_OVERRIDE_KEYS[provider][tier]}`,
};
}
+17 -47
View File
@@ -357,56 +357,27 @@ review. The user turns this off only by asking explicitly
```bash
# Codex preflight: one block (functions sourced here don't persist to later blocks).
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
_gstack_helper_error=""
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
if [ "$_CODEX_CFG" = "disabled" ]; then
_CODEX_MODE="disabled"
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
_CODEX_MODE="under_codex"
elif ! command -v codex >/dev/null 2>&1; then
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
elif [ -n "$_gstack_helper_error" ]; then
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
else
# The free sandbox check runs before the paid model probe. Probe code 2 means
# the CLI cannot execute at all, a different fix from an unusable model.
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
if [ "$_CODEX_MP" -eq 3 ]; then
_CODEX_MODE="sandbox_unavailable"
elif [ "$_CODEX_MP" -eq 2 ]; then
_CODEX_MODE="broken_install"
elif [ "$_CODEX_MP" -eq 4 ]; then
_CODEX_MODE="quota_exhausted"
elif [ "$_CODEX_MP" -ne 0 ]; then
_CODEX_MODE="model_unusable"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
_CODEX_MODE="unverified"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
_CODEX_MODE="unverified (rate_limited)"
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
if [ "$_OUTSIDE_CFG" = disabled ]; then
echo 'CODEX_MODE: disabled'
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
else
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
fi
exit 78
fi
); then
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
else
echo 'CODEX_MODE: under_current_harness'
fi
echo "CODEX_MODE: $_CODEX_MODE"
```
Branch on the echoed `CODEX_MODE`:
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the reviewer invocation; record disabled coverage as directed below; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and construct the prompt below, then follow **Native fallback**. Conflicting inherited harness markers are not grounds to guess another provider.
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
**Outcome routing:** Follow the row for the current result. After an invocation, route its result
again. Leave only after recording disabled/unavailable coverage, or after
@@ -505,8 +476,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+2 -3
View File
@@ -993,7 +993,7 @@ else
fi
```
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
Declined: skip both voices. Non-ready: retain the repair notice, use only the native voice, and record `outside_status: unavailable` even if it succeeds. The invocation rechecks the harness before spawning.
@@ -1050,8 +1050,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+18 -48
View File
@@ -296,56 +296,27 @@ review. The user turns this off only by asking explicitly
```bash
# Codex preflight: one block (functions sourced here don't persist to later blocks).
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
_gstack_helper_error=""
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
if [ "$_CODEX_CFG" = "disabled" ]; then
_CODEX_MODE="disabled"
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
_CODEX_MODE="under_codex"
elif ! command -v codex >/dev/null 2>&1; then
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
elif [ -n "$_gstack_helper_error" ]; then
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
else
# The free sandbox check runs before the paid model probe. Probe code 2 means
# the CLI cannot execute at all, a different fix from an unusable model.
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
if [ "$_CODEX_MP" -eq 3 ]; then
_CODEX_MODE="sandbox_unavailable"
elif [ "$_CODEX_MP" -eq 2 ]; then
_CODEX_MODE="broken_install"
elif [ "$_CODEX_MP" -eq 4 ]; then
_CODEX_MODE="quota_exhausted"
elif [ "$_CODEX_MP" -ne 0 ]; then
_CODEX_MODE="model_unusable"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
_CODEX_MODE="unverified"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
_CODEX_MODE="unverified (rate_limited)"
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
if [ "$_OUTSIDE_CFG" = disabled ]; then
echo 'CODEX_MODE: disabled'
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
else
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
fi
exit 78
fi
); then
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
else
echo 'CODEX_MODE: under_current_harness'
fi
echo "CODEX_MODE: $_CODEX_MODE"
```
Branch on the echoed `CODEX_MODE`:
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider.
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
**Disabled is a terminal branch for this section.** If the preflight prints
`CODEX_MODE: disabled`, persist `outside_status: disabled` with the guarded
@@ -439,8 +410,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
@@ -483,7 +453,7 @@ Immediately before dispatching, check the preflight result again. On
do not dispatch. Otherwise, use this fallback for missing/broken CLI, failed
authentication/model selection, a failed preflight, or a failed outside invocation.
The disabled branch never reaches this fallback.
On `CODEX_MODE: under_codex`, report the setup repair and
On `CODEX_MODE: under_current_harness`, report the setup repair and
`outside_status: unavailable`, run no outside CLI, and use the native subagent below.
A native result never supplies outside coverage.
+19 -47
View File
@@ -799,56 +799,27 @@ review. The user turns this off only by asking explicitly
```bash
# Codex preflight: one block (functions sourced here don't persist to later blocks).
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
_gstack_helper_error=""
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
if [ "$_CODEX_CFG" = "disabled" ]; then
_CODEX_MODE="disabled"
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
_CODEX_MODE="under_codex"
elif ! command -v codex >/dev/null 2>&1; then
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
elif [ -n "$_gstack_helper_error" ]; then
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
else
# The free sandbox check runs before the paid model probe. Probe code 2 means
# the CLI cannot execute at all, a different fix from an unusable model.
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
if [ "$_CODEX_MP" -eq 3 ]; then
_CODEX_MODE="sandbox_unavailable"
elif [ "$_CODEX_MP" -eq 2 ]; then
_CODEX_MODE="broken_install"
elif [ "$_CODEX_MP" -eq 4 ]; then
_CODEX_MODE="quota_exhausted"
elif [ "$_CODEX_MP" -ne 0 ]; then
_CODEX_MODE="model_unusable"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
_CODEX_MODE="unverified"
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
_CODEX_MODE="unverified (rate_limited)"
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
if [ "$_OUTSIDE_CFG" = disabled ]; then
echo 'CODEX_MODE: disabled'
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
else
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
fi
exit 78
fi
); then
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
else
echo 'CODEX_MODE: under_current_harness'
fi
echo "CODEX_MODE: $_CODEX_MODE"
```
Branch on the echoed `CODEX_MODE`:
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the reviewer invocation; record disabled coverage as directed below; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and construct the prompt below, then follow **Native fallback**. Conflicting inherited harness markers are not grounds to guess another provider.
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
**Outcome routing:** Pick exactly one row from this table, finish that row's
steps, then leave Outside Voice. Missing reviewer coverage is non-blocking;
@@ -944,8 +915,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
@@ -1474,6 +1444,8 @@ without adding or strengthening them in the question or descriptions. A test
required before editing one function does not make every independent lane wait.
A next-step answer approves no implementation change.
When the user chooses C: **Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
## Learning hooks
In finish step 6, keep the working plan/approvals fixed. Review operational learnings
@@ -592,6 +592,8 @@ without adding or strengthening them in the question or descriptions. A test
required before editing one function does not make every independent lane wait.
A next-step answer approves no implementation change.
When the user chooses C: {{IMPLEMENTATION_MODEL_HANDOFF}}
## Learning hooks
In finish step 6, keep the working plan/approvals fixed. Review operational learnings
+3 -3
View File
@@ -650,7 +650,7 @@ Sanitize every query before it leaves the machine: strip hostnames, IPs, file pa
## Step 4: Critical pass (core review)
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
Step 4 is read-only: defer charters, setup and probes to Step 4.7.
From the installed /review SKILL.md's directory, choose one path:
@@ -658,9 +658,9 @@ From the installed /review SKILL.md's directory, choose one path:
- If the caller directory is prefixed `gstack-review`, use `../gstack-qa/sections/exploratory.md` instead and read it in full.
- If neither layout applies, report an unresolved QA installation as a setup blocker; do not guess another path.
Use this host's installation, never the product tree. If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
Resolve QA's `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory.
QA's `sections/...` and `templates/...` paths resolve from installed QA SKILL.md, not the caller or product directory.
Apply both checklist passes in order: CRITICAL, then INFORMATIONAL. Respect its suppressions.
+58
View File
@@ -0,0 +1,58 @@
import { readFileSync, writeFileSync, appendFileSync } from 'node:fs';
import { collectFreshnessSources, type Fetcher } from '../lib/model-policy-freshness-sources';
import { assessFreshness, freshnessCatalogIdentity, renderFreshnessReport, type FreshnessObservation } from '../lib/model-policy-freshness';
import { freshnessGitHub } from '../lib/model-policy-freshness-github';
import { publishFreshness, type FreshnessGitHub } from '../lib/model-policy-freshness-publication';
export async function runModelPolicyFreshness(options: {
env?: NodeJS.ProcessEnv; fetcher?: Fetcher; github?: FreshnessGitHub; now?: () => string; publish?: boolean;
}) {
const env = options.env ?? process.env;
const now = options.now ?? (() => new Date().toISOString());
const startedAt = now();
const catalog = freshnessCatalogIdentity(readFileSync(new URL('../lib/model-catalog.ts', import.meta.url), 'utf8'));
const sources = await collectFreshnessSources(options.fetcher);
const observation: FreshnessObservation = {
catalog,
checkedAt: now(), run: { id: env.GITHUB_RUN_ID ?? '0', attempt: Number(env.GITHUB_RUN_ATTEMPT ?? '1'), startedAt, headSha: env.GITHUB_SHA ?? '0'.repeat(40) }, sources,
};
let report = assessFreshness(observation);
let publication;
if (options.publish) {
if (!env.GITHUB_REF) throw new Error('publication-ref-missing');
if (env.GITHUB_REF === 'refs/heads/main' && (!env.GITHUB_REPOSITORY || !env.GH_TOKEN || !env.GITHUB_RUN_ID || !env.GITHUB_SHA)) throw new Error('github-publication-prerequisites-missing');
try {
const github = options.github ?? (env.GITHUB_REF === 'refs/heads/main' ? freshnessGitHub(env.GITHUB_REPOSITORY!, env.GH_TOKEN!, options.fetcher) : null);
publication = github ? await publishFreshness(observation, github, env.GITHUB_REF) : { action: 'report-only' as const };
if ('report' in publication && publication.report) report = publication.report;
} catch (error) {
const reason = error instanceof Error && /^[a-z0-9: /.-]{1,160}$/i.test(error.message) ? error.message : 'publication-unavailable';
publication = { action: 'failed' as const, error: reason };
report = { ...report, status: 'unknown/source-unavailable', complete: false, errors: [...report.errors, `publication:${reason}`] };
}
}
return { schemaVersion: 1, observation, report, publication };
}
async function main(): Promise<number> {
const args = process.argv.slice(2);
if (args.some(arg => !['--publish', '--help'].includes(arg))) { console.error('Usage: bun scripts/model-policy-freshness.ts [--publish]'); return 2; }
if (args.includes('--help')) {
console.log('Check four official public model recommendation/lifecycle sources without paid calls. Default: report-only. --publish requires main/default-branch identity and GITHUB_TOKEN-backed GH_TOKEN; no catalog or pin writes. JSON goes to stdout. Optional MODEL_FRESHNESS_REPORT writes the same artifact; GITHUB_STEP_SUMMARY receives the human report.');
return 0;
}
const result = await runModelPolicyFreshness({ publish: args.includes('--publish') });
const output = `${JSON.stringify(result, null, 2)}\n`;
if (process.env.MODEL_FRESHNESS_REPORT) writeFileSync(process.env.MODEL_FRESHNESS_REPORT, output, { mode: 0o600 });
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${renderFreshnessReport(result.report)}\n`);
console.log(output.trimEnd());
if (result.publication?.action.startsWith('rejected-') || result.publication?.action === 'failed') return 1;
return result.report.status === 'unknown/source-unavailable' || result.report.status === 'stale' ? 1 : 0;
}
if (import.meta.main) {
main().then(code => { process.exitCode = code; }).catch(() => {
console.error('Model-policy freshness or report delivery failed. Confirm the tracking issue state; do not infer freshness from this run. Check the workflow logs, official sources, issue permissions/state, and docs/troubleshooting.md#model-policy-freshness.');
process.exitCode = 1;
});
}
+3 -3
View File
@@ -4,6 +4,7 @@ import * as path from 'node:path';
import { discoverTemplates, includesSkill } from './discover-skills';
import { externalSkillName, extractNameAndDescription } from './external-skill-names';
import { getHostConfig } from '../hosts';
import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration';
const args = process.argv.slice(2);
const value = (flag: string): string => {
@@ -155,12 +156,11 @@ for (const name of hostEntries.filter(name => name.startsWith('gstack-') && !sou
const old = path.join(namespace, 'gstack-claude');
const nextSkill = path.join(namespace, 'gstack-claude-code');
const renameFiles = ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts'];
const runtimeSkill = path.join(runtime, 'SKILL.md');
const runtimeBanner = !fs.existsSync(runtimeSkill) || fs.readFileSync(runtimeSkill, 'utf8').includes('<!-- AUTO-GENERATED from');
const rename = owned(old) && (!exists(nextSkill) || owned(nextSkill))
&& (!exists(runtime)?.isSymbolicLink() || linkIsOurs(runtime))
&& runtimeBanner && renameFiles.every(rel => {
&& runtimeBanner && CLAUDE_CODE_RUNTIME_FILES.every(rel => {
const parent = path.join(runtime, path.dirname(rel));
return exists(path.join(source, rel)) && (!exists(parent)?.isSymbolicLink() || linkIsOurs(parent));
});
@@ -172,7 +172,7 @@ if (rename) {
for (const rel of ['SKILL.md', 'agents/openai.yaml']) checkAtomicCopy(path.join(next, rel), physicalNamespace, 'rename skill write', true);
}
if (exists(runtime) && physical(runtime) !== source) {
for (const rel of renameFiles) {
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
const dest = path.join(runtime, rel);
if (exists(dest) && physical(dest) === physical(path.join(source, rel))) continue;
checkAtomicCopy(dest, physicalNamespace, 'rename runtime write');
+51 -2
View File
@@ -4,6 +4,8 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { ALL_MODEL_NAMES, resolveModel, type Model } from './models';
import { ModelPolicyError, resolvePlanReviewModel, selectionRepair, type ModelSelection } from '../lib/model-policy';
import { emitModelPolicyNotice } from '../lib/model-policy-notice';
export interface CodexGenerationModelResolution {
model: Model;
@@ -181,6 +183,45 @@ export function resolveCodexRuntimeModel(opts: {
return { kind: opts.kind, model: CODEX_DEFAULT_MODEL, source: `gstack default (no model in ${stripControl(configPath)})` };
}
export interface CodexPlanReviewSelection extends CodexRuntimeModelSelection {
selection: ModelSelection;
/** Source-aware repair for this record, never a lower-priority setting. */
repair: string;
}
/**
* Role-bearing Codex selection: the plan-review policy ranks request, env,
* tier config and catalog. Host mode keeps the legacy native resolver,
* labelled as delegated so its origin stays honest.
*/
export function resolveCodexPlanReviewModel(opts: {
kind: CodexInvocationKind;
explicit?: string;
env?: Record<string, string | undefined>;
codexHome?: string;
home?: string;
/** Target repository; its project settings can route to a custom endpoint. */
cwd?: string;
}): CodexPlanReviewSelection {
const env = opts.env ?? process.env;
const selection = resolvePlanReviewModel({ provider: 'openai', requestedModel: opts.explicit || undefined, env, cwd: opts.cwd });
const repair = stripControl(selectionRepair(selection).join('; '));
if (selection.status === 'selected') {
const model = selection.requestedModel;
if (!CODEX_RUNTIME_MODEL_PATTERN.test(model)) {
throw new Error(`Invalid Codex model '${sanitize(model)}' from ${sanitize(selection.source.label)}. ${repair}`);
}
return { kind: opts.kind, model, source: stripControl(selection.source.label), selection, repair };
}
const native = resolveCodexRuntimeModel({ kind: opts.kind, env, codexHome: opts.codexHome, home: opts.home });
return { ...native, source: stripControl(`${selection.source.label}; ${native.source}`), selection, repair: repair || REPAIR };
}
function policyFailure(error: unknown): string {
if (!(error instanceof ModelPolicyError)) return (error as Error).message;
return [`${error.problem} ${error.cause}`.trim(), ...error.repair.map(step => `Repair: ${step}`), `Docs: ${error.docs}`].join('\n');
}
function readArg(name: string): string | undefined {
const exact = process.argv.indexOf(name);
if (exact >= 0) return process.argv[exact + 1];
@@ -193,7 +234,15 @@ if (import.meta.main) {
try {
const runtime = readArg('--runtime');
if (runtime !== undefined) {
if (runtime !== 'exec' && runtime !== 'review') throw new Error('Usage: --runtime exec|review [--explicit <model>]');
if (runtime !== 'exec' && runtime !== 'review') throw new Error('Usage: --runtime exec|review [--explicit <model>] [--role plan-review]');
const role = readArg('--role');
if (role !== undefined) {
if (role !== 'plan-review') throw new Error('Usage: --role plan-review');
const record = resolveCodexPlanReviewModel({ kind: runtime, explicit: readArg('--explicit'), cwd: readArg('--cwd') });
emitModelPolicyNotice(record.selection);
process.stdout.write(`${record.model}\t${record.source}\t${record.selection.tier}\t${record.repair}\n`);
process.exit(0);
}
const selection = resolveCodexRuntimeModel({ kind: runtime, explicit: readArg('--explicit') });
process.stdout.write(`${selection.model}\t${selection.source}\n`);
process.exit(0);
@@ -209,7 +258,7 @@ if (import.meta.main) {
// so a hostile CODEX_HOME cannot smuggle tabs/newlines into the TSV contract.
process.stdout.write(`${result.model}\t${stripControl(result.source)}\n`);
} catch (error) {
process.stderr.write(`${(error as Error).message}\n`);
process.stderr.write(`${policyFailure(error)}\n`);
process.exit(1);
}
}
+4 -2
View File
@@ -57,9 +57,11 @@ export const OPENAI_LITMUS_CHECKS = [
export const CODEX_WEB_SEARCH_FLAG = `-c 'web_search="cached"'`;
/**
* Default model for gstack-owned Codex invocations when nothing else chooses.
* Historical no-role Codex default and the pinned eval ruler some tests import.
* It does not follow the role catalog in lib/model-catalog.ts: plan-review
* calls resolve their model through lib/model-policy.ts at invocation time.
*
* The runtime model is resolved per invocation kind by
* A no-role runtime model is resolved per invocation kind by
* `_gstack_codex_select_model exec|review` (bin/gstack-codex-probe, backed by
* resolveCodexRuntimeModel in scripts/resolve-codex-generation-model.ts):
* explicit request, then GSTACK_CODEX_MODEL, then Codex config.toml (`model`;
+2 -2
View File
@@ -841,7 +841,7 @@ printf 'DESIGN_BRIEF=%s\\n' "$_DESIGN_BRIEF"
Write the product brief to that path; remember its absolute path across fresh Bash calls. Neither voice inherits context: give both the same brief. Include its complete contents in the outside prompt file for Codex, along with the design-direction request below; substitute its shell-quoted absolute path for the literal <prepared-prompt-file> in the invocation. Keep your draft direction out of both prompts; give the native Agent its absolute path (the product brief's path, not the Codex prompt file). Never paste brief text into shell source.` : ''}
**Check ${outsideVoiceFor(ctx).label} availability:**
${outsideVoicePreflight(ctx, { disabledBehavior: 'opt-in', acceptedOnly: isDesignConsultation })}
${outsideVoicePreflight(ctx, { disabledBehavior: 'opt-in', acceptedOnly: isDesignConsultation, ...(isPlanDesignReview ? { role: 'plan-review' as const } : {}) })}
${isDesignConsultation ? 'Non-ready CLI: retain its repair notice and use only the native voice. The invocation deliberately rechecks the harness before spawning; native success never replaces external coverage.' : 'Declined: skip both voices. Non-ready: retain the repair notice, use only the native voice, and record `outside_status: unavailable` even if it succeeds. The invocation rechecks the harness before spawning.'}
@@ -853,7 +853,7 @@ Prompt (include the actual plan/product/frontend source context, not only file p
"${codexPrompt}"
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, reasoningEffort, ...(isDesignConsultation ? { purpose: 'design-direction' as const } : {}) })}
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, reasoningEffort, ...(isDesignConsultation ? { purpose: 'design-direction' as const } : {}), ...(isPlanDesignReview ? { role: 'plan-review' as const } : {}) })}
2. **${outsideVoiceFor(ctx).nativeLabel} design subagent** (Agent tool, \`run_in_background: false\`; await its result):
"${subagentPrompt}"
+4 -2
View File
@@ -27,6 +27,7 @@ import { generateReviewDashboard, generatePlanFileReviewReport } from './review-
import { generatePlanReviewApprovalCheck, generateExitPlanModeGate, generatePlanCompletionAuditShip, generatePlanCompletionGateShip, generatePlanCompletionAuditReview, generatePlanVerificationExec } from './plan-gates';
import { generateAntiShortcutClause, generateSpecReviewLoop, generateBenefitsFrom } from './spec-review';
import { generateCodexSecondOpinion, generateAdversarialStep, generateCodexPlanReview, generateCodexDocReview } from './outside-voice-steps';
import { generateImplementationModelHandoff } from './plan-review';
import { generateScopeDrift, generateCrossReviewDedup, generateSharedCodeReuse } from './review-scope';
import { generateSlugEval, generateSlugSetup, generateBaseBranchDetect, generateDeployBootstrap, generateQAMethodology, generateCoAuthorTrailer, generateChangelogWorkflow, generateCodexWebSearchFlag, generateCodexModelConfigFlag, generateCodexReviewModelConfigFlag, generateClaudeModelFlag, generateSetupCommand } from './utility';
import { generateLearningsSearch, generateLearningsLog } from './learnings';
@@ -57,8 +58,8 @@ export const RESOLVERS: Record<string, ResolverFn> = {
NATIVE_LABEL: (ctx) => outsideVoiceFor(ctx).nativeLabel,
OUTSIDE_PROVIDER: (ctx) => outsideVoiceFor(ctx).id,
HOST_ID: (ctx) => ctx.host,
OUTSIDE_PREFLIGHT: (ctx, args) => outsideVoicePreflight(ctx, { disabledBehavior: args?.[0] === 'opt-in' ? 'opt-in' : 'codex-only' }),
OUTSIDE_INVOCATION: (ctx, args) => outsideVoiceInvocation(ctx, { timeoutMs: args?.[0] === 'spec' ? 120000 : 540000, gate: args?.[0] === 'spec' ? 'spec' : 'review', reasoningEffort: args?.[0] === 'spec' ? 'medium' : 'high' }),
OUTSIDE_PREFLIGHT: (ctx, args) => outsideVoicePreflight(ctx, { disabledBehavior: args?.[0] === 'opt-in' ? 'opt-in' : 'codex-only', ...(args?.[0] === 'autoplan' || args?.[1] === 'plan-review' ? { role: 'plan-review' as const } : {}) }),
OUTSIDE_INVOCATION: (ctx, args) => outsideVoiceInvocation(ctx, { timeoutMs: args?.[0] === 'spec' ? 120000 : 540000, gate: args?.[0] === 'spec' ? 'spec' : 'review', reasoningEffort: args?.[0] === 'spec' ? 'medium' : 'high', ...(args?.[0] === 'autoplan' || args?.[0] === 'spec' ? { role: 'plan-review' as const } : {}) }),
OUTSIDE_PROVENANCE: (ctx, args) => outsideVoiceProvenance(ctx, args?.[0] ?? ctx.skillName),
SLUG_EVAL: generateSlugEval,
SLUG_SETUP: generateSlugSetup,
@@ -125,6 +126,7 @@ export const RESOLVERS: Record<string, ResolverFn> = {
SCOPE_DRIFT: generateScopeDrift,
DEPLOY_BOOTSTRAP: generateDeployBootstrap,
CODEX_PLAN_REVIEW: generateCodexPlanReview,
IMPLEMENTATION_MODEL_HANDOFF: generateImplementationModelHandoff,
CODEX_DOC_REVIEW: generateCodexDocReview,
PLAN_COMPLETION_AUDIT_SHIP: generatePlanCompletionAuditShip,
PLAN_COMPLETION_GATE_SHIP: generatePlanCompletionGateShip,
+3 -3
View File
@@ -416,7 +416,7 @@ completed output. Use private temporary paths, with no background jobs.` : `Run
Finish a failed attempt's termination before fallback; consume only its completed
output. No background jobs or shared temporary paths.`}
${outsideVoiceInvocation(ctx, { timeoutMs: 300000 })}
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, role: 'plan-review' })}
Present the full output verbatim:
@@ -457,7 +457,7 @@ Immediately before dispatching, check the preflight result again. On
do not dispatch. Otherwise, use this fallback for missing/broken CLI, failed
authentication/model selection, a failed preflight${needsApprovalReadiness ? ' (including harness mismatch)' : ''}, or a failed outside invocation.
The disabled branch never reaches this fallback.
${needsApprovalReadiness ? '' : `On \`CODEX_MODE: ${outsideVoiceFor(ctx).id === 'codex' ? 'under_codex' : 'under_current_harness'}\`, report the setup repair and
${needsApprovalReadiness ? '' : `On \`CODEX_MODE: under_current_harness\`, report the setup repair and
\`outside_status: unavailable\`, run no outside CLI, and use the native subagent below.
A native result never supplies outside coverage.`}`}`;
}
@@ -633,7 +633,7 @@ review. The user turns this off only by asking explicitly
**Preflight — decide whether and how the outside voice runs:**
${outsideVoicePreflight(ctx, { disabledBehavior: 'skip-all' })}
${outsideVoicePreflight(ctx, { disabledBehavior: 'skip-all', role: 'plan-review' })}
${codexPlanOutcomeRouting(ctx, ceo, needsApprovalReadiness)}
+15 -6
View File
@@ -44,9 +44,12 @@ if { ${own}; }; then
fi`;
}
export function outsideVoicePreflight(ctx: TemplateContext, opts: { disabledBehavior: 'skip-all' | 'codex-only' | 'opt-in'; acceptedOnly?: boolean; nativeReview?: boolean }): string {
/** Caller-owned workflow role. Only enumerated plan-review generators pass it; everything else stays no-role. */
export type OutsideRole = 'plan-review';
export function outsideVoicePreflight(ctx: TemplateContext, opts: { disabledBehavior: 'skip-all' | 'codex-only' | 'opt-in'; acceptedOnly?: boolean; nativeReview?: boolean; role?: OutsideRole }): string {
const v = outsideVoiceFor(ctx);
if (v.id === 'codex' && opts.disabledBehavior !== 'opt-in') {
if (v.id === 'codex' && opts.disabledBehavior !== 'opt-in' && !opts.role) {
let preflight = outsideVoiceLabels(ctx, codexPreflight({ disabledBehavior: opts.disabledBehavior, nativeReview: opts.nativeReview }))
.replace('```bash\n', `\`\`\`bash\n${runtimeRootPrelude(ctx)}\n`);
if (['plan-eng-review', 'plan-ceo-review'].includes(ctx.skillName)) {
@@ -81,7 +84,7 @@ if [ "$_OUTSIDE_CFG" = disabled ]; then
${ctx.skillName === 'ship' && opts.disabledBehavior === 'opt-in' ? `Ship attempts this optional design check automatically when frontend review applies.
The enabled value above carries that choice. No additional opt-in is needed.
Step 11 keeps its separate outside-review switch.
\`CODEX_MODE\` reports provider availability, not user consent; here the provider is **${v.label}**.` : `The historical \`CODEX_MODE\` variable describes **${v.label}** availability here.`} Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair ${v.label}; authentication failure: run \`${v.id === 'codex' ? 'codex login' : 'claude auth login'}\`. ${opts.disabledBehavior === 'skip-all' ? 'Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer.' : opts.disabledBehavior === 'codex-only' ? 'Disabled skips only the outside CLI; retain the native pass.' : ctx.skillName === 'ship' ? '' : 'Honor this caller’s existing opt-in/skip choice.'} ${opts.disabledBehavior === 'skip-all' ? 'Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled.' : opts.disabledBehavior === 'codex-only' ? 'Non-ready means missing outside coverage. Keep the required native pass without duplicating it.' : 'Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback.'} Never substitute another external provider.`;
\`CODEX_MODE\` reports provider availability, not user consent; here the provider is **${v.label}**.` : `The historical \`CODEX_MODE\` variable describes **${v.label}** availability here.`} ${opts.role ? 'Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation.' : 'Authentication and configured model validity are checked by the actual invocation, without overriding either.'} Missing/broken CLI: install or repair ${v.label}; authentication failure: run \`${v.id === 'codex' ? 'codex login' : 'claude auth login'}\`. ${opts.disabledBehavior === 'skip-all' ? 'Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer.' : opts.disabledBehavior === 'codex-only' ? 'Disabled skips only the outside CLI; retain the native pass.' : ctx.skillName === 'ship' ? '' : 'Honor this caller’s existing opt-in/skip choice.'} ${opts.disabledBehavior === 'skip-all' ? 'Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled.' : opts.disabledBehavior === 'codex-only' ? 'Non-ready means missing outside coverage. Keep the required native pass without duplicating it.' : 'Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback.'} Never substitute another external provider.`;
}
export interface OutsideCommandOptions {
@@ -97,6 +100,8 @@ export interface OutsideCommandOptions {
/** Creative proposals retain the recommendation gate with task-specific wording. */
purpose?: 'design-direction';
nativeAlreadyRequired?: boolean;
/** Resolve the plan-review policy once in the invocation shell and bind it through probe and dispatch. */
role?: OutsideRole;
}
/** One self-contained shell body. No shell functions/variables survive between blocks. */
@@ -108,15 +113,19 @@ export function outsideVoiceCommand(ctx: TemplateContext, opts: OutsideCommandOp
const codex = opts.structuredBase
? `codex review --base ${sh(opts.structuredBase)} -c "sandbox_mode=\\"\${_GSTACK_CODEX_SANDBOX:?}\\"" ${CODEX_REVIEW_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} < /dev/null`
: `codex exec - -C "$_REPO_ROOT" -s "\${_GSTACK_CODEX_SANDBOX:?}" ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT"`;
const kind = opts.structuredBase ? 'review' : 'exec';
const codexSelection = opts.role
? `source "${bin}/gstack-codex-probe" && _gstack_codex_role_ready ${kind} "$_REPO_ROOT" || exit $?`
: `source "${bin}/gstack-codex-probe" && _gstack_codex_select_model ${kind} || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1`;
const invocation = v.id === 'codex'
? `source "${bin}/gstack-codex-probe" && _gstack_codex_select_model ${opts.structuredBase ? 'review' : 'exec'} || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
? `${codexSelection}
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper ${Math.ceil(opts.timeoutMs / 1000)} ${codex} >"$_OUTSIDE_TMP/${opts.structuredBase ? 'text' : 'events'}" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
${opts.structuredBase ? 'cat "$_OUTSIDE_TMP/text"' : 'cat "$_OUTSIDE_TMP/text" 2>/dev/null || tail -n 20 "$_OUTSIDE_TMP/events"'}`
: `_OUTSIDE_EXIT=0
"${bin}/gstack-claude-code" --cwd "$_REPO_ROOT" --access ${opts.access ?? 'none'} --timeout-ms ${opts.timeoutMs} <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/result.json" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
"${bin}/gstack-claude-code" --cwd "$_REPO_ROOT" --access ${opts.access ?? 'none'} --timeout-ms ${opts.timeoutMs}${opts.role ? ` --role ${opts.role}` : ''} <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/result.json" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
# Preserve session/usage/modelUsage from this JSON; multiple models have no invented primary.
cat "$_OUTSIDE_TMP/result.json" || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; }
if [ "$_OUTSIDE_EXIT" -eq 0 ]; then
+7
View File
@@ -0,0 +1,7 @@
/** Plan-approval handoff: recommend the runtime-resolved implementation model, never switch to it. */
import { toShellPath, type TemplateContext } from './types';
export function generateImplementationModelHandoff(ctx: TemplateContext): string {
const provider = ctx.host === 'claude' ? ' --provider anthropic' : ctx.host === 'codex' ? ' --provider openai' : '';
return `**Implementation model:** relay the model and source from \`"${toShellPath(ctx.paths.binDir)}/gstack-models" resolve --role implementation${provider}\`${provider ? '' : ' (one per provider)'}. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.`;
}
+3 -3
View File
@@ -228,12 +228,12 @@ ${setup ? 'Read `sections/browser-setup.md` in full unless already completed;\n'
export function generateQAReviewPreflight(ctx: TemplateContext): string {
sectionPath(ctx, 'qa', 'exploratory');
return `> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
return `> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
${ctx.skillName === 'review' ? 'Step 4 is read-only: defer charters, setup and probes to Step 4.7.\n' : ''}
{{QA_RESOURCE:exploratory}}
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
Resolve QA's \`sections/...\` and \`templates/...\` paths from that installed QA SKILL.md directory, not the caller or product directory.`;
QA's \`sections/...\` and \`templates/...\` paths resolve from installed QA SKILL.md, not the caller or product directory.`;
}
export function generateQAReview(ctx: TemplateContext): string {
+1
View File
@@ -226,6 +226,7 @@ export const FREE_ONLY_PR_FILES = [
'.github/workflows/dependency-review.yml',
'.github/workflows/free-tests.yml',
'.github/workflows/make-pdf-gate.yml',
'.github/workflows/model-policy-freshness.yml',
'.github/workflows/native-qualification.yml',
'.github/workflows/osv-scanner.yml',
'.github/workflows/platform-qualification.yml',
+3 -3
View File
@@ -488,12 +488,12 @@ Never overwrite another run's reports. Batch only independent Reads.
**1. Load methods before any QA or explicit-verification probe.**
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
From the installed /ship SKILL.md's directory, Read `../qa/sections/exploratory.md` in full. If the caller directory is prefixed `gstack-ship`, use `../gstack-qa/sections/exploratory.md` instead. Use this host's installation, never the product tree. If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
Resolve QA's `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory.
QA's `sections/...` and `templates/...` paths resolve from installed QA SKILL.md, not the caller or product directory.
**2. List required checks.**
Run the shared preflight; start its smoke guard once. Guard every smoke probe. For browsers, Read QA's `sections/browser-setup.md` for report-only rules.
+1
View File
@@ -869,6 +869,7 @@ Add to the standard template:
`/autoplan` for the full review gauntlet).
- **For implementation:** the issue itself is the handoff. The implementer can
open it and execute without re-asking the user.
**Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
- **`/ship` integration:** when `/ship` opens a PR for a worktree that contains
a `/spec` archive (frontmatter `spec_issue_number: <N>`) AND the PR delivers
the full spec (acceptance criteria checked off per `/ship`'s existing
+1
View File
@@ -474,6 +474,7 @@ Add to the standard template:
`/autoplan` for the full review gauntlet).
- **For implementation:** the issue itself is the handoff. The implementer can
open it and execute without re-asking the user.
{{IMPLEMENTATION_MODEL_HANDOFF}}
- **`/ship` integration:** when `/ship` opens a PR for a worktree that contains
a `/spec` archive (frontmatter `spec_issue_number: <N>`) AND the PR delivers
the full spec (acceptance criteria checked off per `/ship`'s existing
+2 -3
View File
@@ -149,7 +149,7 @@ else
fi
```
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
Write the prompt with the exact redaction-approved spec bytes using the Write tool; never shell-interpolate the raw draft. Keep hard delimiters and this boundary:
@@ -178,8 +178,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
_gstack_codex_sandbox_preflight >/dev/null || exit 1
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
_gstack_codex_first_use_notice
_OUTSIDE_EXIT=0
_gstack_codex_timeout_wrapper 120 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="medium"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
+1 -1
View File
@@ -60,7 +60,7 @@ dispatch. `spec-quality-gate-secret-sink.test.ts` enforces this.
**Dispatch (only when redaction passes):** No reviewer preflight/dispatch before the redaction decision. When blocked, STOP before Phase 5 and all downstream sinks. On --no-gate record skipped after redaction succeeds.
{{OUTSIDE_PREFLIGHT:opt-in}}
{{OUTSIDE_PREFLIGHT:opt-in:plan-review}}
Write the prompt with the exact redaction-approved spec bytes using the Write tool; never shell-interpolate the raw draft. Keep hard delimiters and this boundary:
+41 -2
View File
@@ -2,7 +2,8 @@ import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { migrateClaudeCodeSkills } from '../lib/claude-code-migration';
import { spawnSync } from 'node:child_process';
import { CLAUDE_CODE_RUNTIME_FILES as runtimeFiles, migrateClaudeCodeSkills } from '../lib/claude-code-migration';
const ROOT = path.resolve(import.meta.dir, '..');
const banner = '<!-- AUTO-GENERATED from SKILL.md.tmpl — do not edit directly -->\n<!-- Regenerate: bun run gen:skill-docs -->';
@@ -15,7 +16,7 @@ function fixture() {
const root = path.join(dir, 'checkout'); const home = path.join(dir, 'home');
const codex = path.join(home, 'custom-codex', 'skills');
const kiro = path.join(home, '.kiro', 'skills');
for (const rel of ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts']) put(path.join(root, rel), rel);
for (const rel of runtimeFiles) put(path.join(root, rel), rel);
const oldRender = path.join(root, '.agents', 'skills', 'gstack-claude');
put(path.join(oldRender, 'SKILL.md'), skill('gstack-claude'));
fs.mkdirSync(codex, { recursive: true }); fs.mkdirSync(kiro, { recursive: true });
@@ -40,6 +41,44 @@ function fixture() {
}
describe('Claude wrapper installed-name migration', () => {
test('copied policy tools and wrapper load without the source checkout', () => {
const f = fixture();
try {
for (const rel of runtimeFiles) fs.copyFileSync(path.join(ROOT, rel), path.join(f.root, rel));
put(path.join(f.codex, 'gstack-claude', 'SKILL.md'), skill('gstack-claude'));
expect(f.run({ copy: true })).toEqual({ migrated: 1, pending: [] });
fs.renameSync(f.root, `${f.root}-unavailable`);
const runtime = path.join(f.codex, 'gstack');
const env = {
...process.env, HOME: f.home, USERPROFILE: f.home,
GSTACK_STATE_ROOT: path.join(f.home, 'state'), CODEX_HOME: path.dirname(f.codex),
CLAUDE_CONFIG_DIR: path.join(f.home, '.claude'),
GSTACK_CLAUDE_MODEL: '', GSTACK_CODEX_MODEL: '', ANTHROPIC_BASE_URL: '', OPENAI_BASE_URL: '',
CLAUDE_CODE_USE_BEDROCK: '', CLAUDE_CODE_USE_VERTEX: '', CLAUDE_CODE_USE_FOUNDRY: '',
BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0',
};
const set = spawnSync('bash', [path.join(runtime, 'bin/gstack-config'), 'set', 'plan_review_tier', 'smart'], {
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
});
expect(set.status).toBe(0);
const inspect = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-models'), 'resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], {
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
});
expect(inspect.status).toBe(0);
expect(JSON.parse(inspect.stdout).selections[0].requestedModel).toBe('claude-opus-5-5');
const wrapper = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-claude-code'), '--help'], {
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
});
expect(wrapper.status).toBe(1);
expect(JSON.parse(wrapper.stdout).error.code).toBe('arguments');
const gate = spawnSync(process.execPath, ['-e', 'const {classifyOutsideReview} = await import(process.argv[1]); console.log(classifyOutsideReview({text:"",gate:"review",exit:1}).verdict)', path.join(runtime, 'lib/outside-review-result.ts')], {
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
});
expect(gate.status).toBe(0);
expect(gate.stdout.trim()).toBe('unavailable');
} finally { fs.rmSync(f.dir, { recursive: true, force: true }); }
});
test('migrates existing Codex and Kiro installs independently of the selected setup host', () => {
const f = fixture();
try {
+10 -8
View File
@@ -937,15 +937,17 @@ describe('codex broken-install detection (#2742)', () => {
// hand-maintained copy that isn't resolver-generated — it must capture the
// probe's exit code and route 2 to its own broken-install arm, or /autoplan
// prints the wrong remedy for a broken binary.
test('autoplan preflight (tmpl + rendered) captures the probe exit and routes 2 to broken-install', () => {
for (const rel of ['autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md']) {
const raw = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
const src = rel.endsWith('.tmpl') ? raw.replace('{{OUTSIDE_PREFLIGHT:autoplan}}', RESOLVERS.OUTSIDE_PREFLIGHT({ host: 'claude', paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: rel }, ['autoplan'])) : raw;
expect(src).toContain('_gstack_codex_model_probe; _CODEX_MP=$?');
expect(src).toMatch(/_CODEX_MP" -eq 2/);
expect(src).toContain('binary cannot run');
expect(src).not.toContain('elif ! _gstack_codex_model_probe');
test('autoplan (tmpl + rendered): availability pays for no probe; the invocation probe exit, 2 included, ends it', () => {
const ctx = { host: 'claude' as const, paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: 'autoplan/SKILL.md.tmpl' };
const rendered = ['autoplan/SKILL.md', ...['ceo', 'design', 'dx', 'eng'].map(phase => `autoplan/sections/${phase}-phase.md`)]
.map(rel => fs.readFileSync(path.join(ROOT, rel), 'utf-8')).join('\n');
for (const src of [RESOLVERS.OUTSIDE_PREFLIGHT(ctx, ['autoplan']) + RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan']), rendered]) {
expect(src).toContain('_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?');
expect(src).not.toMatch(/_gstack_codex_model_probe|_CODEX_MP/);
}
const role = fs.readFileSync(PROBE, 'utf-8').match(/_gstack_codex_role_ready\(\) \{[\s\S]*?\n\}/)![0];
expect(role).toContain('_gstack_codex_model_probe "$1"');
expect(role.trim().split('\n').at(-2)!.trim()).toBe('_gstack_codex_model_probe "$1"');
});
test('the preflight resolver routes exit 2 to broken_install', () => {
+258
View File
@@ -0,0 +1,258 @@
/**
* Bounded multi-signature Codex probe cache (stubbed codex, no API spend):
* 16 entries, atomic publication, same-signature serialization with a
* bounded lock, abandoned-owner recovery, and unverified (never a duplicate
* paid probe) on lock timeout or unreadable cache.
*/
import { describe, expect, test } from 'bun:test';
import { spawn, spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
const ROOT = path.resolve(import.meta.dir, '..');
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
const STUB = `#!/usr/bin/env bash
[ "$1" = sandbox ] && exit 0
printf '%s\\n' "$*" >> "$STUB_LOG"
[ -n "\${STUB_SLEEP:-}" ] && sleep "$STUB_SLEEP"
case "\${STUB_MODE:-ok}" in
ok) echo OK; exit 0 ;;
model400) echo 'ERROR: {"type":"error","status":400,"error":{"message":"The model is not supported when using Codex with a ChatGPT account."}}' >&2; exit 1 ;;
quota) echo "ERROR: You've hit your usage limit. Try again at 3 AM." >&2; exit 1 ;;
esac
`;
interface Fixture { home: string; codexHome: string; state: string; log: string; cache: string; locks: string; env: Record<string, string> }
function fixture(): Fixture {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-probe-cache-'));
const bin = path.join(home, 'bin');
const codexHome = path.join(home, '.codex');
const state = path.join(home, 'state');
for (const dir of [bin, codexHome, state]) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(bin, 'codex'), STUB, { mode: 0o755 });
fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n');
fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}');
const log = path.join(home, 'stub.log');
return {
home, codexHome, state, log,
cache: path.join(state, '.codex-model-probe'),
locks: path.join(state, '.codex-model-probe.locks'),
env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: home, CODEX_HOME: codexHome, GSTACK_HOME: state, STUB_LOG: log, _TEL: 'off' },
};
}
function probe(f: Fixture, env: Record<string, string> = {}, call = '_gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"') {
const r = spawnSync('bash', ['-c', `source "${PROBE}"\n${call}`], { env: { ...f.env, ...env }, encoding: 'utf8', timeout: 20000 });
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status };
}
function probeAsync(f: Fixture, env: Record<string, string>): Promise<string> {
return new Promise((resolve, reject) => {
const child = spawn('bash', ['-c', `source "${PROBE}"\n_gstack_codex_model_probe; echo "rc=$?"`], { env: { ...f.env, ...env } });
let out = '';
child.stdout.on('data', chunk => { out += chunk; });
const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('probe hung')); }, 30000);
child.on('close', () => { clearTimeout(timer); resolve(out); });
});
}
const calls = (f: Fixture) => (fs.existsSync(f.log) ? fs.readFileSync(f.log, 'utf8').split('\n').filter(Boolean).length : 0);
const entries = (f: Fixture) => fs.readFileSync(f.cache, 'utf8').split('\n').filter(line => /^MODEL_\S+ \d+ \d+$/.test(line));
const model = (name: string) => ({ GSTACK_CODEX_MODEL: name });
const withFixture = (body: (f: Fixture) => void | Promise<void>) => async () => {
const f = fixture();
try { await body(f); } finally { fs.rmSync(f.home, { recursive: true, force: true }); }
};
describe('bounded multi-signature probe cache', () => {
test('A -> B -> A makes exactly two paid probes', withFixture((f) => {
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK');
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK');
const again = probe(f, model('gpt-a'));
expect(again.stdout).toContain('MODEL_OK (cached)');
expect(calls(f)).toBe(2);
expect(entries(f)).toHaveLength(2);
expect(fs.existsSync(f.locks) ? fs.readdirSync(f.locks) : []).toEqual([]);
}));
test('concurrent same-signature misses share one probe', withFixture(async (f) => {
const outputs = await Promise.all([0, 1, 2, 3].map(() => probeAsync(f, { ...model('gpt-same'), STUB_SLEEP: '1' })));
expect(calls(f)).toBe(1);
for (const out of outputs) expect(out).toMatch(/MODEL_OK(?: \(cached\))?\nrc=0/);
expect(outputs.filter(out => out.includes('(cached)'))).toHaveLength(3);
expect(entries(f)).toHaveLength(1);
}));
test('the 17th signature evicts the oldest entry; the file never exceeds 16', withFixture((f) => {
for (let i = 1; i <= 17; i++) expect(probe(f, model(`gpt-m${i}`)).stdout).toContain('MODEL_OK');
expect(entries(f)).toHaveLength(16);
expect(probe(f, model('gpt-m17')).stdout).toContain('MODEL_OK (cached)');
expect(probe(f, model('gpt-m2')).stdout).toContain('MODEL_OK (cached)');
expect(calls(f)).toBe(17);
expect(probe(f, model('gpt-m1')).stdout).not.toContain('(cached)');
expect(calls(f)).toBe(18);
expect(entries(f)).toHaveLength(16);
}));
test('expired entries are pruned and never hit; TTLs and negative classifications are unchanged', withFixture((f) => {
probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
probe(f, model('gpt-good'));
const now = Math.floor(Date.now() / 1000);
const lines = fs.readFileSync(f.cache, 'utf8').split('\n').filter(Boolean)
.map(line => line.replace(/^(MODEL_UNUSABLE) (\d+)/, (_, s) => `${s} ${now - 901}`));
fs.writeFileSync(f.cache, `${lines.join('\n')}\n`);
expect(probe(f, { ...model('gpt-good') }).stdout).toContain('MODEL_OK (cached)');
const bad = probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
expect(bad.stdout).toContain('MODEL_UNUSABLE\n');
expect(bad.stdout).toContain('rc=1');
expect(calls(f)).toBe(3);
expect(probe(f, { ...model('gpt-bad'), STUB_MODE: 'ok' }).stdout).toContain('MODEL_UNUSABLE (cached)');
expect(calls(f)).toBe(3);
const future = fs.readFileSync(f.cache, 'utf8').replace(/^MODEL_OK \d+/m, `MODEL_OK ${now + 999}`);
fs.writeFileSync(f.cache, future);
expect(probe(f, model('gpt-good')).stdout).not.toContain('(cached)');
}));
test('quota detail lines survive among other entries and are relayed on a hit', withFixture((f) => {
probe(f, model('gpt-x'));
expect(probe(f, { ...model('gpt-q'), STUB_MODE: 'quota' }).stdout).toContain('rc=4');
probe(f, model('gpt-y'));
const hit = probe(f, model('gpt-q'));
expect(hit.stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)');
expect(hit.stdout).toContain("ERROR: You've hit your usage limit. Try again at 3 AM.");
expect(hit.stdout).toContain('rc=4');
expect(probe(f, { ...model('gpt-q'), GSTACK_CODEX_PROBE_RETRY: '1' }).stdout).toContain('MODEL_OK\n');
expect(calls(f)).toBe(4);
expect(entries(f)).toHaveLength(3);
}));
test('a pre-v2 one-entry cache migrates by TTL: fresh honored, expired re-probed, stray lines ignored', withFixture((f) => {
probe(f);
const [status, ts, sig] = entries(f)[0].split(' ');
expect(status).toBe('MODEL_OK');
fs.writeFileSync(f.cache, `MODEL_OK ${ts} ${sig}\nan unprefixed legacy line\n`);
expect(probe(f).stdout).toContain('MODEL_OK (cached)');
fs.writeFileSync(f.cache, `MODEL_OK ${Number(ts) - 3700} ${sig}\n`);
expect(probe(f).stdout).not.toContain('(cached)');
expect(calls(f)).toBe(2);
}));
test('an interrupted writer leaves a truncated line and temp file that never count as a hit', withFixture((f) => {
probe(f, model('gpt-a'));
fs.appendFileSync(f.cache, 'MODEL_OK 17');
fs.writeFileSync(`${f.cache}.tmp.99999`, 'MODEL_OK 1 1\n');
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK (cached)');
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK\n');
expect(fs.readFileSync(f.cache, 'utf8')).not.toContain('MODEL_OK 17\n');
expect(entries(f)).toHaveLength(2);
}));
test('config, auth and binary changes still invalidate a cached entry', withFixture((f) => {
probe(f);
const future = Date.now() / 1000 + 10;
fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future);
expect(probe(f).stdout).not.toContain('(cached)');
fs.utimesSync(path.join(f.codexHome, 'auth.json'), future + 5, future + 5);
expect(probe(f).stdout).not.toContain('(cached)');
expect(probe(f, { CODEX_API_KEY: 'k-test' }).stdout).not.toContain('(cached)');
expect(calls(f)).toBe(4);
}));
});
describe('probe lock recovery and failure', () => {
const sigOf = (f: Fixture) => {
probe(f);
const sig = entries(f)[0].split(' ')[2];
fs.rmSync(f.cache);
fs.rmSync(f.log);
return sig;
};
test('a dead owner on this host is recovered at once, without a duplicate probe', withFixture((f) => {
const sig = sigOf(f);
const lock = path.join(f.locks, sig);
fs.mkdirSync(lock, { recursive: true });
const dead = spawnSync('bash', ['-c', 'echo $$'], { timeout: 5000 }).stdout.toString().trim();
fs.writeFileSync(path.join(lock, 'owner'), `${dead} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`);
const r = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '3' });
expect(r.stdout).toContain('MODEL_OK\n');
expect(calls(f)).toBe(1);
expect(fs.existsSync(lock)).toBe(false);
}));
test('a live owner past the stale bound is abandoned; within it the waiter is unverified without probing', withFixture((f) => {
const sig = sigOf(f);
const lock = path.join(f.locks, sig);
fs.mkdirSync(lock, { recursive: true });
const now = Math.floor(Date.now() / 1000);
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now}\n`);
const busy = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' });
expect(busy.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (another probe of this model still holds its lock)');
expect(busy.stdout).toContain('rc=0 state=inconclusive');
expect(calls(f)).toBe(0);
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now - 120}\n`);
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
expect(calls(f)).toBe(1);
}));
test('a lock without its owner file is the creation window until it is 5 seconds old', withFixture((f) => {
const sig = sigOf(f);
const lock = path.join(f.locks, sig);
fs.mkdirSync(lock, { recursive: true });
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' }).stdout).toContain('MODEL_PROBE_INCONCLUSIVE');
const old = Date.now() / 1000 - 30;
fs.utimesSync(lock, old, old);
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
expect(calls(f)).toBe(1);
}));
test('an unreadable cache or uncreatable lock is unverified and launches no paid probe', withFixture((f) => {
fs.mkdirSync(f.cache);
const dir = probe(f);
expect(dir.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
expect(dir.stdout).toContain('rc=0 state=inconclusive');
fs.rmdirSync(f.cache);
fs.writeFileSync(f.cache, '');
fs.chmodSync(f.cache, 0o000);
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
fs.chmodSync(f.cache, 0o644);
fs.rmSync(f.cache);
fs.writeFileSync(f.locks, 'not a directory');
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
expect(calls(f)).toBe(0);
}));
test('an owner record that cannot be written is lock I/O failure: unverified, no paid probe, no stray lock', withFixture((f) => {
fs.mkdirSync(f.locks, { recursive: true });
const r = probe(f, {}, '_gstack_codex_select_model exec 2>/dev/null; umask 0777; _gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"');
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot record ownership of the probe lock');
expect(r.stdout).toContain('rc=0 state=inconclusive');
expect(calls(f)).toBe(0);
expect(fs.readdirSync(f.locks)).toEqual([]);
expect(fs.existsSync(f.cache)).toBe(false);
expect(probe(f).stdout).toContain('MODEL_OK\n');
expect(calls(f)).toBe(1);
}));
test('a cache write failure keeps the real probe result', withFixture((f) => {
fs.chmodSync(f.state, 0o500);
try {
const r = probe(f, { STUB_MODE: 'model400' });
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
fs.chmodSync(f.state, 0o700);
fs.mkdirSync(f.locks);
fs.writeFileSync(path.join(f.locks, 'write'), 'blocks the write lock');
const real = probe(f, { STUB_MODE: 'model400' });
expect(real.stdout).toContain('MODEL_UNUSABLE\n');
expect(real.stdout).toContain('rc=1');
expect(fs.existsSync(f.cache)).toBe(false);
} finally { fs.chmodSync(f.state, 0o700); }
}));
test('sourcing the probe still leaves the caller shell untouched', () => {
const r = spawnSync('bash', ['-c', `B=$(set -o; trap -p; echo "$IFS"); source "${PROBE}"; A=$(set -o; trap -p; echo "$IFS"); [ "$A" = "$B" ] && echo CLEAN`], { encoding: 'utf8', timeout: 5000 });
expect(r.stdout.trim()).toBe('CLEAN');
});
});
+1 -1
View File
@@ -90,7 +90,7 @@ describe('under-codex wiring renders (#2519)', () => {
test('all three codexPreflight consumers render the probe', () => {
for (const file of [
path.join(ROOT, 'ship', 'sections', 'adversarial.md'),
path.join(ROOT, 'plan-ceo-review', 'sections', 'review-sections.md'),
path.join(ROOT, 'review', 'sections', 'adversarial.md'),
path.join(ROOT, 'document-release', 'sections', 'release-body.md'),
]) {
const rendered = fs.readFileSync(file, 'utf-8');
+42
View File
@@ -0,0 +1,42 @@
import { readFileSync } from 'node:fs';
import { FRESHNESS_SOURCES, parseOfficialSource, sha256, type SourceId, type SourceResult } from '../../lib/model-policy-freshness-sources';
import type { CatalogIdentity, FreshnessObservation } from '../../lib/model-policy-freshness';
export const freshnessCatalog: CatalogIdentity = {
sha256: sha256('catalog-v1'), sourceSha256: sha256('catalog-source-v1'), version: 1,
models: [
{ provider: 'anthropic', tier: 'frontier', modelId: 'claude-fable-5-1' },
{ provider: 'anthropic', tier: 'smart', modelId: 'claude-opus-5-5' },
{ provider: 'openai', tier: 'frontier', modelId: 'gpt-6-astra' },
{ provider: 'openai', tier: 'smart', modelId: 'gpt-6.1-sol' },
],
};
export function freshnessFixture(id: SourceId): string {
return readFileSync(new URL(`./model-policy-freshness/${id}.md`, import.meta.url), 'utf8');
}
export function freshnessObservation(id = '100', at = '2026-10-07T12:00:00.000Z', overrides: Partial<Record<SourceId, string | null>> = {}): FreshnessObservation {
const sources: SourceResult[] = FRESHNESS_SOURCES.map(source => {
const text = overrides[source.id] === undefined ? freshnessFixture(source.id) : overrides[source.id];
return text === null ? { id: source.id, error: 'http-503' } : {
id: source.id, receipt: { id: source.id, url: source.url, finalUrl: source.url, sha256: sha256(text!), bytes: Buffer.byteLength(text!), parsed: parseOfficialSource(source.id, text!) },
};
});
return { catalog: structuredClone(freshnessCatalog), checkedAt: at, run: { id, attempt: 1, startedAt: new Date(Date.parse(at) - 1000).toISOString(), headSha: 'a'.repeat(40) }, sources };
}
export function retirementFixture(): string {
return freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027', 'claude-fable-5-1 | Deprecated | October 1, 2026 | November 1, 2026');
}
export function freshnessReplacementObservation(): FreshnessObservation {
const observation = freshnessObservation('101', '2026-10-08T12:00:00.000Z', {
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
});
observation.catalog.models[0].modelId = 'claude-fable-5-2';
observation.catalog.sha256 = 'b'.repeat(64);
observation.catalog.sourceSha256 = 'c'.repeat(64);
return observation;
}
@@ -0,0 +1,16 @@
# Model deprecations
Partner-operated platforms (Amazon Bedrock and Google Cloud) set their own retirement schedules.
## Model status
| API model name | Current state | Deprecated | Tentative retirement date |
| --- | --- | --- | --- |
| claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027 |
| claude-opus-5-5 | Active | N/A | Not sooner than September 22, 2027 |
| claude-mythos-preview | Deprecated | June 9, 2026 | To be announced |
| claude-opus-4-1-20250805 | Retired | June 5, 2026 | August 5, 2026 |
## Deprecation history
Amazon Bedrock partner discussion mentions `claude-fable-5-1` retirement; this is not a public API lifecycle row.
@@ -0,0 +1,17 @@
# Models overview
## Compare models
If you're unsure which model to use, start with [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) for most workloads. Use [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) for demanding reasoning and long-horizon agentic work.
| Feature | Claude Fable 5.1 | Claude Opus 5.5 | Claude Sonnet 5.5 | Claude Haiku 5.5 |
| --- | --- | --- | --- | --- |
| Model page | [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) | [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) | [Claude Sonnet 5.5](https://platform.claude.com/docs/en/models/sonnet-5-5/overview) | [Claude Haiku 5.5](https://platform.claude.com/docs/en/models/haiku-5-5/overview) |
| Claude API ID | `claude-fable-5-1` | `claude-opus-5-5` | `claude-sonnet-5-5` | `claude-haiku-5-5` |
| Amazon Bedrock ID | `anthropic.claude-fable-5-1` | `anthropic.claude-opus-5-5` | `anthropic.claude-sonnet-5-5` | `anthropic.claude-haiku-5-5` |
* **Pricing:** prompt cache reads cost 10% of the base input price (2.5% on Claude Fable 5.1 and Claude Mythos 5.1, 5% on Claude Opus 5.5). See Pricing for per-platform pricing. Incidental ID: `claude-mythos-5-1`.
## Using the Models API
Legacy `claude-fable-5` and unrelated `claude-mythos-5-1` are not recommendations.
@@ -0,0 +1,24 @@
# Deprecations
This page lists all API deprecations, along with recommended replacements.
## Upcoming deprecations
### 2026-10-01: GPT-5.3-Codex
| Shutdown date | Model / system | Recommended replacement |
| --- | --- | --- |
| Apr 1, 2027 | `gpt-5.3-codex` | `gpt-6-sol` |
## Past deprecations
| Shutdown date | Model snapshot | Substitute model |
| --- | --- | --- |
| Aug 5, 2026 | `gpt-old` | `gpt-6-astra` |
| 2026‑03‑26 | `gpt-unicode-date` | `gpt-6-astra` |
| 2026-02-12 | `gpt-iso-date` | `gpt-6-astra` |
| at earliest 2024-06-13 | `gpt-date-floor` | `gpt-6-astra` |
| 2026-09-24 | Videos API | --- |
| 2024-01-04 | `/v1/edits` | `/v1/chat/completions` |
Incidental replacement text `gpt-6-astra` is not a deprecated model row.
+13
View File
@@ -0,0 +1,13 @@
# Models
## Featured models
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Start here for complex reasoning and coding.
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Balance intelligence and cost.
- [GPT-6 Luna](/api/docs/models/gpt-6-luna.md): Optimize cost-sensitive, high-volume workloads.
## Browse our full catalog of models
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Our most capable model for the most demanding work.
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Near-Astra performance for complex work at a lower cost.
- [GPT-9 Preview](/api/docs/models/gpt-9-preview.md): Incidental full-catalog listing, not a featured recommendation.
+115
View File
@@ -0,0 +1,115 @@
{
"source": "ship-exploratory-late-input, mtb1007a, 2026-10-07; public synthetic fixture events only",
"events": [
{
"type": "assistant",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
"content": [
{
"type": "tool_use",
"id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
"name": "Read",
"input": {
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/exploratory.md"
},
"caller": {
"type": "direct"
}
}
]
}
},
{
"type": "user",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"content": [
{
"tool_use_id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
"type": "tool_result",
"content": "1\t<!-- AUTO-GENERATED from exploratory.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Shared exploratory QA\n4\t\n5\tThe **caller** (/qa, /qa-only, /review or /ship) owns decisions, tests, fixes and publication. Discovery writes only reports/evidence\n6\tand owned fixture state; no workflows, framework installs or publication.\n7\t\n8\tComplete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation.\n9\t1. Read `sections/scope.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and select the surfaces.\n10\t2. Read the selected surface methods below in full.\n11\t\n12\t**Functional surfaces:**\n13\tRead `sections/system-functional.md` in full.\n14\t\n15\t**Browser surfaces only:**\n16\tRead `sections/qa-patterns.md` in full.\n17\t\n18\tMissing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks. Report QA setup blockers.\n19\t\n20\t## 1. Charter and preflight\n21\t\n22\tReuse resolved REPORT_DIR; otherwise own a fresh `.gstack/qa-reports` subdirectory.\n23\tWrite a **charter** per behavior: contract, risk, entrypoint, isolation, exit condition, source, commands and inputs. Save charters as Markdown in the report.\n24\t\n25\tFor /review and /ship, no plan/server is required.\n26\tStop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces).\n27\tExplicit plan checks and revalidation remain required beyond this smoke budget.\n28\tFor /qa and /qa-only:\n29\t- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900.\n30\t- Functional Full, Quick and Regression have no default total timer.\n31\tSet SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.\n32\tWithout a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.\n33\tStop when scoped contracts are tested or blocked.\n34\tClocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.\n35\tNames used below (quote every path):\n36\t- PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run.\n37\t- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.\n38\t- DEADLINE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-deadline`, the deadline guard: `start` creates DEADLINE_FILE, `status` prints `remainingMs` and `expired`, and `run` executes one command, stopping it at the deadline.\n39\t- EVIDENCE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence`, the functional evidence recorder: `capture` runs one command, stores its exit code, stdout and stderr under `PROBE_DIR/.qa-evidence/NNN/`, and prints status `complete`, `incomplete` or `sensitive`.\n40\t\n41\tStart once before baseline: `bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]` if bounded.\n42\tEARLIER_UTC = caller's absolute deadline, if set.\n43\tFunctional: `bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS`.\n44\tUnbounded: use `--timeout-ms MS` instead of `--deadline DEADLINE_FILE`.\n45\t--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of `PROBE_DIR/.qa-evidence/NNN/observation.json` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.\n46\tBounded browsers: `bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS`. No detached probes.\n47\tNever reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.\n48\t\n49\t## 2. Probe loop\n50\t\n51\tEach probe is one native command/interaction plus checks, excluding bookkeeping.\n52\tNever batch probes.\n53\t\n54\t1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.\n55\t2. **Decide whether another probe is needed.** If bounded, run `bun DEADLINE_TOOL status DEADLINE_FILE`.\n56\t If expired or no safe next probe remains, STOP exploration; write the report (\u00a74), not a checkpoint.\n57\t **Publish before probing.** Create (browser) or compose (functional) `exploration-NNN.json` in the probe directory, beside its deadline if bounded, with exactly four top-level fields:\n58\t observationCommand: last completed probe's full outer command, including guard.\n59\t observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text.\n60\t hypothesis: why nextCommand. nextCommand: exact command/request, guarded if bounded.\n61\t PrLine truncated
}
]
}
},
{
"type": "assistant",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
"content": [
{
"type": "tool_use",
"id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
"name": "Bash",
"input": {
"command": "bun /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence capture /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports 001 --public --deadline /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports/deadline.json -- bun scripts/probe.ts 3",
"description": "Capture happy-path probe for input 3"
},
"caller": {
"type": "direct"
}
}
]
}
},
{
"type": "user",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"content": [
{
"tool_use_id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
"type": "tool_result",
"content": "{\"id\":\"probe-ba7e52d4-ad96-4df9-97d0-6dd79e52f184\",\"charter\":\"happy\",\"input\":\"3\",\"snapshot\":\"936ff5b3b13172fb2357a120804c1cf5cac68d911c82cae5bce630f04dd62b57\",\"status\":\"pass\",\"stdout\":\"6\\n\",\"stderr\":\"\",\"exit\":0}\n\nQA_EVIDENCE {\"producer\":\"gstack-qa-evidence\",\"version\":1,\"action\":\"capture\",\"id\":\"001\",\"status\":\"complete\",\"sha256\":\"135ce1a8f804e3c6d953e0f284ab4e1e7fe5ded56f3a35706ea73f630876d0be\",\"exitCode\":0,\"signal\":null,\"publicOutput\":true,\"startedAt\":\"2026-10-07T22:13:30.620Z\",\"completedAt\":\"2026-10-07T22:13:30.675Z\",\"durationMs\":55,\"remainingMs\":182686,\"next\":\"Another probe requires a checkpoint anchored on capture 001: add --after 001 --hypothesis 'TEXT' before --. To stop exploring, run none.\"}",
"is_error": false
}
]
},
"tool_use_result": {
"interrupted": false
}
},
{
"type": "assistant",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"id": "msg_011CfoeRZyA2j8av6NchLTmY",
"content": [
{
"type": "tool_use",
"id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
"name": "Read",
"input": {
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/system-functional.md"
},
"caller": {
"type": "direct"
}
}
]
}
},
{
"type": "user",
"parent_tool_use_id": null,
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
"message": {
"content": [
{
"tool_use_id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
"type": "tool_result",
"content": "1\t<!-- AUTO-GENERATED from system-functional.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Functional QA with repository-native tools\n4\t\n5\tUse documented repository commands, CLI/API clients and job/queue tools, not a new\n6\tharness or browser substitution.\n7\t\n8\t## Functional modes\n9\t\n10\tFor /qa and /qa-only, within the selected scope:\n11\t- **Full** (default): cover every applicable documented contract below.\n12\t- **Quick** (`--quick`): check success and the highest-risk changed edge; mark other\n13\t contracts not run.\n14\t- **Regression** (`--regression <previous-report>`): before probes, read the supplied\n15\t functional report and linked replay evidence. A missing, unreadable or wrong-target\n16\t baseline blocks regression mode. A browser-only `baseline.json` is not a functional\n17\t baseline. Re-establish owned setup; replay prior failed probes against the documented\n18\t expectation, never recorded buggy output, then check changed adjacent behavior.\n19\t Preserve the prior report; report fixed, still failing and new findings separately.\n20\t Missing safe replay inputs block affected probes, never count as passes.\n21\t\n22\tMixed runs apply each surface's mode separately. /review and /ship retain their caller's\n23\tbounded smoke and explicit plan checks, not Full exploration.\n24\t\n25\t## Contract map\n26\t\n27\tRecord each contract/source, isolated setup, exact probe, expectation and outcome:\n28\tpass/fail/blocked/not run/inconclusive/not applicable (reason).\n29\t\n30\t| Contract | Observe |\n31\t|---|---|\n32\t| Successful execution | Expected return/output and final business effect, not just launch/acceptance |\n33\t| Invalid/missing input | Declared rejection, correct status and no forbidden state change |\n34\t| Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary |\n35\t| CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes |\n36\t| State transitions | Initial, intermediate and completed/failed states and their permitted transitions |\n37\t| Timeout/cancellation | Deadline, partial state, termination of owned work and recovery |\n38\t| Retry | Attempts/backoff/terminal state promised by the repository; no unbounded retry |\n39\t| Duplicates/idempotency | Repeated request/event and number of durable effects under the documented guarantee |\n40\t| Concurrency/order | Controlled competing operations in both relevant completion orders; final invariant |\n41\t| Partial-failure recovery | Interrupt after an effect, restart/replay, inspect completion/dead-letter state and duplicates |\n42\t\n43\tDo not impose universal exactly-once delivery. Separate acceptance, enqueue, processing,\n44\tretry/dead-letter and final effect; 2xx is not completion. Expected rejection/injected\n45\tfailure may pass; a missing service preventing execution blocks coverage.\n46\t\n47\t## Execute and retain evidence\n48\t\n49\t1. Apply the shared isolation/permission preflight. Verify cwd, command, environment\n50\t NAMES and safe reset; use synthetic data/credentials.\n51\t2. Follow the shared exploratory loop's order and written checkpoints.\n52\t For every probe, inspect initial/final durable state and retain exit/status and\n53\t stdout/stderr separately without masking failure.\n54\t3. On timeout, retain partial output/state and stop only owned work. Record setup errors\n55\t and untested contracts; never patch product code to hide missing prerequisites.\n56\t4. Record exact command or method/path/headers/body, setup/reset, expected contract/source,\n57\t observed output/state, revision/runtime, evidence paths and limits. Secrets are referenced\n58\t only by environment name. Disclose replay limits caused by redaction.\n59\t5. Use `templates/functional-report-template.md` relative to the installed QA SKILL.md.\n60\t Preserve evidence before owned cleanup and disclose leftovers. Return to the caller\n61\t without expanding discovery authority.\n62\t"
}
]
}
}
]
}
+3
View File
@@ -123,6 +123,8 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'],
'test/fixtures/module-size-ratchet.json': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
'test/fixtures/module-size/**': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
'test/fixtures/model-policy-freshness.ts': ['test/model-policy-freshness.test.ts', 'test/model-policy-workflow.test.ts'],
'test/fixtures/model-policy-freshness/**': ['test/fixtures/model-policy-freshness.ts'],
'test/fixtures/native-auq-boxed-full-body-sep21.json': ['test/auq-native-capture.test.ts'],
'test/fixtures/native-auq-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
@@ -138,6 +140,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'],
'test/fixtures/plan-floor-edit-wrap-37176835584.json': ['test/plan-floor-edit-wrap.test.ts'],
'test/fixtures/plan-format-kind-notes.json': ['test/plan-format-kind-note.test.ts'],
'test/fixtures/qa-caller-read-probe-mtb1007a.json': ['test/qa-exploratory-callers.test.ts'],
'test/fixtures/plans/autoplan-dashboard.md': ['test/autoplan-init.test.ts', 'test/autoplan-preconfigured-onboarding-ar.test.ts', 'test/autoplan-snapshot.test.ts'],
'test/fixtures/plans/autoplan-focus-appearance-design.md': [],
'test/fixtures/plans/autoplan-focus-appearance.md': [],
+6 -4
View File
@@ -110,6 +110,11 @@ export function callerTools(transcript: unknown[]): CallerTool[] {
return tools;
}
export function callerProbeReadErrors(probe: CallerTool, resources: readonly (readonly [string, CallerTool | undefined])[]): string[] {
const premature = (read: CallerTool) => probe.index <= read.resultIndex || !!probe.messageId && probe.messageId === read.messageId;
return resources.flatMap(([name, read]) => read && premature(read) ? [`probe preceded resource read: ${name}`] : []);
}
export interface CallerProbe {
id: string;
charter: string;
@@ -358,10 +363,7 @@ export function validateCallerEvidence(input: {
else {
checkpointProbes.push({ command: String(tool.input.command), observed: probe, index: tool.index });
if (parentRead && (tool.index <= parentRead.resultIndex || tool.messageId && tool.messageId === parentRead.messageId)) errors.push(`probe preceded parent entrypoint: ${probe.id}`);
for (const id of ['exploratory', 'system-functional']) {
const resource = readOf(`/qa/sections/${id}.md`);
if (resource && (tool.index <= resource.resultIndex || tool.messageId && tool.messageId === resource.messageId)) errors.push(`probe preceded resource read: ${id}`);
}
errors.push(...callerProbeReadErrors(tool, ['exploratory', 'system-functional'].map(id => [id, readOf(`/qa/sections/${id}.md`)] as const)));
if (input.requireGuardedSmoke) {
const command = String(tool.input.command);
const guarded = callerDeadlineCommand(command, deadline);
+238
View File
@@ -0,0 +1,238 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { modelCatalogSha256 } from '../lib/model-catalog';
import { validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy';
const ROOT = path.resolve(import.meta.dir, '..');
const MODELS = path.join(ROOT, 'bin', 'gstack-models');
const CONFIG = path.join(ROOT, 'bin', 'gstack-config');
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
});
function fixture(config?: string) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack models cli '));
roots.push(root);
const state = path.join(root, 'state');
const home = path.join(root, 'home');
fs.mkdirSync(state);
fs.mkdirSync(home);
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
return { root, state, home, env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } as Record<string, string> };
}
function models(args: string[], env: Record<string, string>, cwd = path.dirname(env.HOME)) {
const result = spawnSync(process.execPath, [MODELS, ...args], { encoding: 'utf-8', env, cwd, timeout: 30_000 });
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
function config(args: string[], env: Record<string, string>) {
const result = spawnSync('bash', [CONFIG, ...args], { encoding: 'utf-8', env, timeout: 30_000 });
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
function json(args: string[], env: Record<string, string>, cwd?: string) {
const result = models([...args, '--json'], env, cwd);
expect(result.stderr).toBe('');
return { code: result.code, doc: JSON.parse(result.stdout) };
}
function tree(dir: string): string[] {
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
}
describe('gstack-models inspector', () => {
test('--help explains roles, values, offline behavior, delegation and exits', () => {
const result = models(['--help'], fixture().env);
expect(result.code).toBe(0);
for (const text of ['plan-review', 'implementation', 'frontier | smart | host', 'frontier | smart ', 'offline', 'no paid model call', 'host-controlled', 'Exit: 0', '2 invalid usage', 'schemaVersion 1']) {
expect(result.stdout).toContain(text);
}
});
test('list is the default and emits one JSON document with both roles and tier entries', () => {
const { env, state } = fixture();
const human = models([], env);
expect(human.code).toBe(0);
expect(human.stdout).toContain('plan-review via openai: gpt-6-astra');
expect(human.stdout).toContain('implementation via anthropic: claude-opus-5-5');
expect(human.stdout).toContain('gstack-config set plan_review_tier smart');
expect(human.stderr).toBe('');
expect(models(['list'], env).stdout).toBe(human.stdout);
const { code, doc } = json([], env);
expect(code).toBe(0);
expect(doc).toMatchObject({ schemaVersion: 1, command: 'list', ok: true, offline: true, configPath: path.join(state, 'config.yaml'), errors: [] });
expect(doc.catalog).toMatchObject({ version: 1, sha256: modelCatalogSha256() });
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/anthropic', 'plan-review/openai', 'implementation/anthropic', 'implementation/openai']);
expect(doc.tiers).toHaveLength(4);
expect(doc.settings.plan_review_tier).toEqual({ value: null, effective: 'frontier', origin: 'default' });
expect(doc.settings.model_smart_openai).toEqual({ value: null, effective: 'gpt-6.1-sol', origin: 'default' });
});
test('resolve plan-review returns one tagged selection with its source', () => {
const { env } = fixture('model_frontier_openai: gpt-pinned\n');
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CLAUDE_MODEL: 'claude-ignored' });
expect(code).toBe(0);
expect(doc.selections).toHaveLength(1);
expect(doc.selections[0]).toMatchObject({ status: 'selected', role: 'plan-review', provider: 'openai', tier: 'frontier', requestedModel: 'gpt-pinned', source: { kind: 'config', key: 'model_frontier_openai' } });
const envWins = json(['resolve', '--role=plan-review', '--provider=openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt-env' });
expect(envWins.doc.selections[0]).toMatchObject({ requestedModel: 'gpt-env', source: { kind: 'env', name: 'GSTACK_CODEX_MODEL' } });
});
test('resolve implementation returns both providers unless one is named', () => {
const { env } = fixture('implementation_tier: frontier\n');
const both = json(['resolve', '--role', 'implementation'], env);
expect(both.code).toBe(0);
expect(both.doc.selections.map((s: { provider: string; requestedModel: string }) => [s.provider, s.requestedModel])).toEqual([['anthropic', 'claude-fable-5-1'], ['openai', 'gpt-6-astra']]);
const one = json(['resolve', '--role', 'implementation', '--provider', 'openai'], env);
expect(one.doc.selections).toHaveLength(1);
expect(one.doc.selections[0].provider).toBe('openai');
});
test('host mode is a valid delegated resolution with a null model', () => {
const { env } = fixture('plan_review_tier: host\n');
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
expect(code).toBe(0);
expect(doc.selections[0]).toMatchObject({ status: 'delegated-host', requestedModel: null, source: { kind: 'host', reason: 'host-mode' } });
const human = models(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
expect(human.code).toBe(0);
expect(human.stdout).toContain('host-controlled');
});
test.each([
[['bogus'], "unknown command 'bogus'"],
[['resolve'], 'resolve needs --role'],
[['resolve', '--role', 'coding'], "unknown role 'coding'"],
[['resolve', '--role', 'plan-review'], 'needs --provider'],
[['resolve', '--role', 'plan-review', '--provider', 'claude'], "unknown provider 'claude'"],
[['resolve', '--role', 'implementation', '--provider', 'OpenAI'], "unknown provider 'OpenAI'"],
[['resolve', '--role', 'plan-review', '--role', 'implementation', '--provider', 'openai'], '--role given twice'],
[['resolve', '--role'], '--role needs a value'],
[['list', '--provider', 'openai'], 'unknown argument'],
])('usage error %p exits 2', (args, message) => {
const { env } = fixture();
const human = models(args, env);
expect(human.code).toBe(2);
expect(human.stdout).toBe('');
expect(human.stderr).toContain(message);
const { code, doc } = json(args, env);
expect(code).toBe(2);
expect(doc).toMatchObject({ schemaVersion: 1, ok: false, selections: [] });
expect(doc.errors[0]).toMatchObject({ reason: 'usage' });
expect(doc.errors[0].problem).toContain(message);
});
test('invalid config exits 1 with structured, source-aware repair', () => {
const { env } = fixture('plan_review_tier: fronteir\n');
const { code, doc } = json([], env);
expect(code).toBe(1);
expect(doc).toMatchObject({ ok: false, selections: [], settings: null });
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_plan_review_tier', key: 'plan_review_tier' });
expect(doc.errors[0].repair).toContain('gstack-config unset plan_review_tier');
expect(doc.errors[0].docs).toContain('#model-policy-config');
const human = models(['resolve', '--role', 'implementation'], env);
expect(human.code).toBe(1);
expect(human.stdout).toBe('');
expect(human.stderr).toContain('fix: gstack-config unset plan_review_tier');
});
test('an unusable environment override points at the env var', () => {
const { env } = fixture();
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt 6' });
expect(code).toBe(1);
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', role: 'plan-review', provider: 'openai' });
expect(doc.errors[0].repair[0]).toBe('unset GSTACK_CODEX_MODEL');
});
test('custom routing fails only the affected selections', () => {
const { env } = fixture();
const { code, doc } = json([], { ...env, CLAUDE_CODE_USE_BEDROCK: '1' });
expect(code).toBe(1);
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/openai', 'implementation/openai']);
expect(doc.errors.map((e: { reason: string; role: string }) => `${e.reason}/${e.role}`)).toEqual(['custom_provider_requires_model/plan-review', 'custom_provider_requires_model/implementation']);
});
test('project settings in the working directory are custom routing; detection limits are published', () => {
const { env, root } = fixture();
const project = path.join(root, 'project');
fs.mkdirSync(path.join(project, '.claude'), { recursive: true });
fs.writeFileSync(path.join(project, '.claude', 'settings.local.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } }));
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env, project);
expect(code).toBe(1);
expect(doc.errors[0]).toMatchObject({ reason: 'custom_provider_requires_model', provider: 'anthropic' });
expect(doc.providerDetection.unscanned.anthropic.join(' ')).toContain('server-managed');
expect(json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env).code).toBe(0);
expect(models(['--help'], env).stdout).toContain('cannot see server-managed or MDM policy');
});
test('inspection never writes to the state root or home', () => {
const { root, env } = fixture('plan_review_tier: smart\n');
const before = tree(root);
models([], env);
models(['--json'], env);
models(['resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], env);
models(['resolve', '--role', 'implementation'], env);
expect(tree(root)).toEqual(before);
});
});
describe('gstack-config model policy keys', () => {
test('defaults, set, resolve again and unset restore', () => {
const { env } = fixture();
expect(config(['get', 'plan_review_tier'], env)).toMatchObject({ code: 0, stdout: 'frontier' });
expect(config(['get', 'implementation_tier'], env)).toMatchObject({ code: 0, stdout: 'smart' });
expect(config(['get', 'model_frontier_claude'], env)).toMatchObject({ code: 0, stdout: '' });
expect(config(['defaults'], env).stdout).toContain('plan_review_tier:');
expect(config(['set', 'plan_review_tier', 'smart'], env).code).toBe(0);
expect(config(['set', 'model_smart_openai', ' gpt-custom '], env).code).toBe(0);
expect(config(['get', 'model_smart_openai'], env).stdout).toBe('gpt-custom');
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'smart', requestedModel: 'gpt-custom', source: { kind: 'config' } });
expect(config(['list'], env).stdout).toContain('model_smart_openai: gpt-custom (set)');
expect(config(['unset', 'plan_review_tier'], env).code).toBe(0);
expect(config(['unset', 'model_smart_openai'], env).code).toBe(0);
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'frontier', requestedModel: 'gpt-6-astra', source: { kind: 'catalog' } });
});
test.each([
['plan_review_tier', 'fronteir'],
['plan_review_tier', ' '],
['implementation_tier', 'host'],
['model_frontier_openai', 'gpt 6'],
['model_frontier_claude', 'a'.repeat(513)],
])('invalid %s %p is rejected and the old value kept', (key, value) => {
const { env, state } = fixture();
const good = key.startsWith('model_') ? 'model-1' : 'frontier';
expect(config(['set', key, good], env).code).toBe(0);
const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8');
const result = config(['set', key, value], env);
expect(result.code).toBe(1);
expect(result.stderr).toContain('Existing value left unchanged');
expect(result.stderr).toContain(`gstack-config unset ${key}`);
expect(result.stderr).toContain('#model-policy-config');
expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before);
});
test('set and resolve share one validation rule', () => {
const samples: Array<[PolicyConfigKey, string]> = [
['plan_review_tier', 'frontier'], ['plan_review_tier', 'smart'], ['plan_review_tier', 'host'], ['plan_review_tier', 'Smart'],
['implementation_tier', 'frontier'], ['implementation_tier', 'smart'], ['implementation_tier', 'host'],
['model_frontier_claude', 'claude-opus-4-6[1m]'], ['model_frontier_claude', 'vendor+claude@2026'], ['model_frontier_claude', 'a'.repeat(512)],
['model_frontier_claude', 'a'.repeat(513)], ['model_smart_claude', 'claude;x'], ['model_smart_claude', 'claude x'], ['model_smart_claude', '$(id)'],
['model_frontier_openai', 'gpt-6.1-sol'], ['model_frontier_openai', 'org/model:v1'], ['model_frontier_openai', 'a'.repeat(100)],
['model_frontier_openai', 'a'.repeat(101)], ['model_smart_openai', 'gpt+6'], ['model_smart_openai', 'gpt@6'],
];
const { env } = fixture();
for (const [key, value] of samples) {
const accepted = config(['set', key, value], env).code === 0;
expect({ key, value: value.slice(0, 20), accepted }).toEqual({ key, value: value.slice(0, 20), accepted: validatePolicyValue(key, value) === null });
}
});
});
+245
View File
@@ -0,0 +1,245 @@
import { describe, expect, test, spyOn } from 'bun:test';
import { collectFreshnessSources, fetchOfficialSource, FRESHNESS_BOUNDS, FRESHNESS_SOURCES, lifecycleDate, parseOfficialSource, parseOfficialSourceBounded, withinDeadline } from '../lib/model-policy-freshness-sources';
import { assessFreshness, renderFreshnessReport, storedEvidenceStatus, validEvidence, validLifecycle, validState } from '../lib/model-policy-freshness';
import { freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
describe('official recommendation and lifecycle structures', () => {
test('four official fixtures parse, with Mythos and replacement mentions excluded', () => {
const observation = freshnessObservation();
expect(assessFreshness(observation).status).toBe('current');
const anthropic = parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models'));
expect(anthropic.recommendations).toEqual({ frontier: 'claude-fable-5-1', smart: 'claude-opus-5-5' });
expect(anthropic.lineup).not.toContain('claude-mythos-5-1');
const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle'));
expect(openai.lifecycle?.map(row => row.modelId)).toEqual(['gpt-5.3-codex', 'gpt-old', 'gpt-unicode-date', 'gpt-iso-date', 'gpt-date-floor']);
expect(openai.lifecycle?.find(row => row.modelId === 'gpt-date-floor')).toMatchObject({ deadline: null, supportFloor: '2024-06-13' });
});
test('known-family and actual unknown-family lineup replacements become candidates', () => {
const known = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2').replaceAll('Fable 5.1', 'Fable 5.2');
const knownReport = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': known }));
expect(knownReport.status).toBe('update-candidate');
expect(knownReport.candidates[0].reason).toBe('changed-monitored-family-recommendation');
const unknown = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'mythos-5-1').replaceAll('Fable 5.1', 'Mythos 5.1');
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': unknown }));
expect(report.status).toBe('update-candidate');
expect(report.candidates[0]).toMatchObject({ proposed: 'claude-mythos-5-1', reason: 'unknown-replacement-family-needs-triage' });
expect(report.complete).toBe(true);
});
test('OpenAI full-catalog newer IDs alone are not recommendations', () => {
const original = freshnessFixture('openai-models');
expect(assessFreshness(freshnessObservation()).candidates).toEqual([]);
const changed = original.replaceAll('gpt-6.1-sol', 'gpt-6.2-sol').replaceAll('GPT-6.1 Sol', 'GPT-6.2 Sol');
expect(assessFreshness(freshnessObservation('100', undefined, { 'openai-models': changed })).candidates[0]).toMatchObject({ proposed: 'gpt-6.2-sol' });
});
test('active support floors are never retirement deadlines, even after the floor', () => {
const parsed = parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle'));
expect(parsed.lifecycle?.[0]).toMatchObject({ state: 'active', deadline: null, supportFloor: '2027-09-01' });
expect(assessFreshness(freshnessObservation('100', '2028-10-07T12:00:00.000Z')).state.lifecycle).toEqual([]);
});
test('actual API retirement and removal are urgent, incidental partner prose is ignored', () => {
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
expect(report.status).toBe('update-candidate');
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
expect(renderFreshnessReport(report)).toContain('URGENT');
expect(renderFreshnessReport(report)).toContain('Bedrock/Vertex/custom');
const removed = assessFreshness(freshnessObservation('100', '2026-11-02T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }));
expect(removed.state.lifecycle[0].state).toBe('removed');
});
test('OpenAI shutdown rows apply to the deprecated column, not the replacement', () => {
const text = freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`');
const report = assessFreshness(freshnessObservation('100', undefined, { 'openai-lifecycle': text }));
expect(report.state.lifecycle).toHaveLength(1);
expect(report.state.lifecycle[0]).toMatchObject({ modelId: 'gpt-6-astra', deadline: '2027-04-01' });
});
test('missing required structures and semantic drift fail closed', () => {
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('Claude API ID', 'Google Cloud ID'))).toThrow('missing-api-id-row');
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('for demanding reasoning', 'for unexplained goals'))).toThrow('unrecognized-recommendation');
expect(() => parseOfficialSource('openai-models', freshnessFixture('openai-models').replace('Balance intelligence and cost.', 'Best new model.'))).toThrow('unrecognized-recommendation');
expect(() => parseOfficialSource('openai-lifecycle', '# New deprecations\nNo model tables')).toThrow('missing-or-ambiguous-section');
expect(() => parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', 'gpt-5.3-codex'))).toThrow('unrecognized-deprecated-model-cell');
expect(() => parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle').replace('Active', 'Probably active'))).toThrow('unrecognized-lifecycle-state');
expect(() => lifecycleDate('February 30, 2027')).toThrow('malformed-lifecycle-date');
expect(() => lifecycleDate('tomorrow')).toThrow('malformed-lifecycle-date');
expect(() => parseOfficialSource('anthropic-models', `${freshnessFixture('anthropic-models')}\n## Compare models\nDuplicate`)).toThrow('ambiguous-section');
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('## Using the Models API', 'Use [Other](https://platform.claude.com/docs/en/models/opus-5-5/overview) for demanding reasoning.\n\n## Using the Models API'))).toThrow('unrecognized-recommendation');
});
});
describe('bounded fixed-source fetching', () => {
test('exact byte limit passes and a streamed byte beyond it fails without trusting content-length', async () => {
const atCap = 'x'.repeat(FRESHNESS_BOUNDS.responseBytes);
expect((await fetchOfficialSource('openai-models', async () => new Response(atCap))).bytes).toBe(2_097_152);
await expect(fetchOfficialSource('openai-models', async () => new Response(`${atCap}x`))).rejects.toThrow('response-too-large');
await expect(fetchOfficialSource('openai-models', async () => new Response('x', { headers: { 'content-length': '2097153' } }))).rejects.toThrow('response-too-large');
expect(() => parseOfficialSource('openai-models', `${atCap}x`)).toThrow('response-too-large');
});
test('all redirect destinations are checked; three redirects pass, four fail', async () => {
const credentialUrl = new URL(FRESHNESS_SOURCES[2].url);
credentialUrl.username = 'user';
credentialUrl.password = 'pass';
for (const location of ['https://evil.example/models.md', 'http://developers.openai.com/api/docs/models.md', credentialUrl.href, 'https://developers.openai.com/api/docs/models.md?untrusted=1']) {
let calls = 0;
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location } }); })).rejects.toThrow('redirect-not-allowlisted');
expect(calls).toBe(1);
}
let calls = 0;
expect((await fetchOfficialSource('openai-models', async () => ++calls <= 3 ? new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }) : new Response('data'))).text).toBe('data');
calls = 0;
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }); })).rejects.toThrow('redirect-limit');
expect(calls).toBe(4);
});
test('per-response and whole-check clocks are installed at the exact approved bounds', async () => {
const installed: number[] = [];
const real = globalThis.setTimeout;
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, milliseconds: number) => {
installed.push(milliseconds);
return real(callback, milliseconds === 30_000 ? 5 : milliseconds);
}) as typeof setTimeout);
try {
const result = await collectFreshnessSources(async () => new Promise<Response>(() => {}));
expect(result.every(source => source.error === 'deadline-exceeded')).toBe(true);
expect(installed.filter(ms => ms === 30_000)).toHaveLength(4);
expect(installed).toContain(180_000);
} finally { timers.mockRestore(); }
const controller = new AbortController();
await expect(withinDeadline(new Promise(() => {}), 5, controller)).rejects.toThrow('deadline-exceeded');
expect(controller.signal.aborted).toBe(true);
});
test('only four fixed sources are fetched and failures never become parseable success', async () => {
const urls: string[] = [];
const result = await collectFreshnessSources(async (url, init) => {
urls.push(url);
expect(init.redirect).toBe('manual');
const id = FRESHNESS_SOURCES.find(source => source.url === url)!.id;
return id === 'openai-lifecycle' ? new Response('authentication needed', { status: 403 }) : new Response(freshnessFixture(id));
});
expect(urls.sort()).toEqual(FRESHNESS_SOURCES.map(source => source.url).sort());
expect(result.find(source => source.id === 'openai-lifecycle')?.error).toBe('http-403');
const observation = freshnessObservation();
observation.sources = result;
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
});
test('CPU-bound parsing is terminable and the complete-check deadline stops all blocked parsers', async () => {
let terminations = 0;
const createWorker = () => ({ postMessage() {}, terminate() { terminations++; }, onmessage: null, onerror: null }) as unknown as Worker;
const controller = new AbortController();
const blocked = parseOfficialSourceBounded('openai-models', freshnessFixture('openai-models'), controller.signal, createWorker);
controller.abort();
await expect(blocked).rejects.toThrow('deadline-exceeded');
expect(terminations).toBe(1);
const real = globalThis.setTimeout;
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => real(callback, ms === 180_000 ? 5 : ms)) as typeof setTimeout);
try {
const result = await collectFreshnessSources(async url => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id)), createWorker);
expect(result.every(source => source.error && !source.receipt)).toBe(true);
} finally { timers.mockRestore(); }
expect(terminations).toBe(5);
});
});
describe('trustworthy evidence and deterministic statuses', () => {
test('source-check evidence is separate from catalog qualification and survives artifact expiry', () => {
const report = assessFreshness(freshnessObservation());
expect(validState(report.state)).toBe(true);
expect(validEvidence(report.state.lastSuccess)).toBe(true);
expect(report.state.lastSuccess?.nextCheckDueBy).toBe('2026-10-14T12:00:00.000Z');
expect(report.state.lastSuccess?.sources).toHaveLength(4);
expect(report.state.lastSuccess?.catalog).toEqual(freshnessObservation().catalog);
expect(storedEvidenceStatus(freshnessObservation().catalog, JSON.parse(JSON.stringify(report.state.lastSuccess)), '2026-10-08T12:00:00.000Z')).toBe('current');
expect(renderFreshnessReport(report)).toContain('not automatic liveness monitoring');
});
test('missing/corrupt/mismatched/future evidence is unknown; thirty-day staleness has an exact boundary', () => {
const observation = freshnessObservation();
const evidence = assessFreshness(observation).state.lastSuccess!;
expect(storedEvidenceStatus(observation.catalog, null, observation.checkedAt)).toBe('unknown/source-unavailable');
expect(storedEvidenceStatus({ ...observation.catalog, sha256: 'b'.repeat(64) }, evidence, observation.checkedAt)).toBe('unknown/source-unavailable');
expect(storedEvidenceStatus(observation.catalog, { ...evidence, parserVersion: 99 }, observation.checkedAt)).toBe('unknown/source-unavailable');
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-10-06T12:00:00.000Z')).toBe('unknown/source-unavailable');
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.000Z')).toBe('current');
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.001Z')).toBe('stale');
});
test('partial recovery cannot refresh evidence or erase an unresolved retirement', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const next = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'openai-lifecycle': null }), initial.state);
expect(next.status).toBe('unknown/source-unavailable');
expect(next.state.lastSuccess).toEqual(initial.state.lastSuccess);
expect(next.state.lifecycle).toEqual(initial.state.lifecycle);
const old = assessFreshness(freshnessObservation('102', '2026-11-08T12:00:00.000Z', { 'openai-lifecycle': null }), next.state);
expect(old.evidenceStatus).toBe('stale');
expect(old.status).toBe('unknown/source-unavailable');
const recovered = assessFreshness(freshnessObservation('103', '2026-11-09T12:00:00.000Z'), old.state);
expect(recovered.status).toBe('update-candidate');
expect(recovered.state.lifecycle).toEqual(old.state.lifecycle);
});
test('corrupt duplicate lifecycle keys and source receipts cannot certify current', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
expect(validLifecycle([initial.state.lifecycle[0], initial.state.lifecycle[0]])).toBe(false);
const observation = freshnessObservation();
observation.sources[0].receipt!.finalUrl = 'https://evil.example/docs';
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
const evidence = structuredClone(initial.state.lastSuccess!);
evidence.sources[0] = evidence.sources[1];
expect(validEvidence(evidence)).toBe(false);
expect(validState({ ...initial.state, lastSuccess: evidence })).toBe(false);
});
test('a partial run cannot downgrade an unresolved retirement to legacy or extend its deadline', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const legacy = freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active', 'claude-fable-5-1 | Legacy');
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': legacy, 'openai-lifecycle': null }), initial.state);
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
expect(report.state.lastSuccess).toEqual(initial.state.lastSuccess);
});
test('only a complete check of a reviewed replacement catalog resolves removed identities', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const replacement = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
replacement.catalog.models[0].modelId = 'claude-fable-5-2';
replacement.catalog.sha256 = 'b'.repeat(64);
replacement.sources = freshnessObservation('101', replacement.checkedAt, {
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
}).sources;
const incomplete = structuredClone(replacement);
incomplete.sources[3] = { id: 'openai-lifecycle', error: 'http-503' };
expect(assessFreshness(incomplete, initial.state).state.lifecycle[0].resolvedByCatalog).toBeUndefined();
const complete = assessFreshness(replacement, initial.state);
expect(complete.status).toBe('current');
expect(complete.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
});
test('complete catalog rollback reopens the historical retirement even when the current source row is active', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
expect(replacement.status).toBe('current');
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), replacement.state);
expect(rollback.status).toBe('update-candidate');
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01', firstObservedAt: initial.state.lifecycle[0].firstObservedAt });
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
});
test('source-unavailable catalog rollback reopens history without refreshing prior replacement evidence', () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), replacement.state);
expect(rollback.status).toBe('unknown/source-unavailable');
expect(rollback.state.lastSuccess).toEqual(replacement.state.lastSuccess);
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
});
});
+405
View File
@@ -0,0 +1,405 @@
import { describe, expect, test, spyOn } from 'bun:test';
import { readFileSync } from 'node:fs';
import { MODEL_CATALOG, modelCatalogSha256 } from '../lib/model-catalog';
import { assessFreshness, freshnessCatalogIdentity } from '../lib/model-policy-freshness';
import { FRESHNESS_SOURCES, FRESHNESS_BOUNDS, sha256 } from '../lib/model-policy-freshness-sources';
import { FRESHNESS_ISSUE_AUTHOR, FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, publishFreshness, readFreshnessIssue, writeFreshnessIssue, type FreshnessGitHub, type TrackingIssue } from '../lib/model-policy-freshness-publication';
import { freshnessGitHub } from '../lib/model-policy-freshness-github';
import { runModelPolicyFreshness } from '../scripts/model-policy-freshness';
import { freshnessCatalog, freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
function mockGitHub(initial: (Omit<TrackingIssue, 'author'> & { author?: string })[] = []) {
const issues: TrackingIssue[] = structuredClone(initial).map(issue => ({ author: FRESHNESS_ISSUE_AUTHOR, ...issue }));
const calls: string[] = [];
let identity = { defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 };
const github: FreshnessGitHub = {
async defaultCatalog() { calls.push('catalog'); return identity; },
async listIssues() { calls.push('list'); return structuredClone(issues); },
async readIssue(number) { calls.push('read'); return structuredClone(issues.find(issue => issue.number === number)!); },
async createIssue(body) { calls.push('create'); const issue: TrackingIssue = { number: issues.length + 1, body, state: 'open', author: FRESHNESS_ISSUE_AUTHOR }; issues.push(issue); return structuredClone(issue); },
async updateIssue(number, update) { calls.push('update'); Object.assign(issues.find(issue => issue.number === number)!, update); },
};
return { github, issues, calls, setIdentity(value: typeof identity) { identity = value; } };
}
describe('stable-marker issue ownership and evidence lifecycle', () => {
test('healthy bootstrap creates one closed issue, retains evidence when closed, and repeat publication is idempotent', async () => {
const api = mockGitHub();
const observation = freshnessObservation();
const created = await publishFreshness(observation, api.github, 'refs/heads/main');
expect(created.action).toBe('created');
expect(api.issues).toHaveLength(1);
expect(api.issues[0].state).toBe('closed');
expect(api.issues[0].body).toContain(FRESHNESS_ISSUE_MARKER);
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess?.checkedAt).toBe(observation.checkedAt);
const body = api.issues[0].body;
expect((await publishFreshness(observation, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
expect(api.issues[0].body).toBe(body);
const next = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
expect((await publishFreshness(next, api.github, 'refs/heads/main')).action).toBe('updated');
expect(api.issues).toHaveLength(1);
expect(api.issues[0].state).toBe('closed');
});
test('marker owns the issue, title collisions and human body/disposition text are untouched', async () => {
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
const humanPrefix = 'Human notes: retain this default until qualification.\n';
const humanSuffix = '\nHuman disposition: mitigation pending; do not delete.\n';
const api = mockGitHub([
{ number: 40, body: 'Model-policy freshness: advisory maintenance', state: 'open' },
{ number: 41, body: humanPrefix + initial + humanSuffix, state: 'closed' },
]);
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
expect(api.issues[0].body).toBe('Model-policy freshness: advisory maintenance');
expect(api.issues[1].body.startsWith(humanPrefix)).toBe(true);
expect(api.issues[1].body.endsWith(humanSuffix)).toBe(true);
expect(api.issues[1].state).toBe('open');
expect(readFreshnessIssue(api.issues[1].body).lifecycle[0].state).toBe('deprecated');
});
test('retirement survives source failure and partial recovery; issue never closes on either', async () => {
const api = mockGitHub();
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
const evidence = readFreshnessIssue(api.issues[0].body).state!.lastSuccess;
const failure = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
expect(failure.report?.status).toBe('unknown/source-unavailable');
expect(api.issues[0].state).toBe('open');
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess).toEqual(evidence);
await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'openai-lifecycle': null }), api.github, 'refs/heads/main');
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].deadline).toBe('2026-11-01');
expect(api.issues[0].state).toBe('open');
});
test('both complete and source-unavailable catalog rollbacks reopen the closed replacement issue', async () => {
for (const unavailable of [false, true]) {
const api = mockGitHub();
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
const replacement = freshnessReplacementObservation();
api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 });
await publishFreshness(replacement, api.github, 'refs/heads/main');
expect(api.issues[0].state).toBe('closed');
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBe(replacement.catalog.sha256);
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
const rollback = freshnessObservation('102', '2026-10-09T12:00:00.000Z', unavailable ? { 'anthropic-lifecycle': null, 'openai-lifecycle': null } : {});
const result = await publishFreshness(rollback, api.github, 'refs/heads/main');
expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'update-candidate');
expect(api.issues[0].state).toBe('open');
expect(api.issues).toHaveLength(1);
const stored = readFreshnessIssue(api.issues[0].body);
expect(stored.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
expect(stored.lifecycle[0].resolvedByCatalog).toBeUndefined();
if (unavailable) expect(stored.state?.lastSuccess?.catalog.sha256).toBe(replacement.catalog.sha256);
}
});
test('corrupt state salvages independent lifecycle history and requires explicit manual repair', async () => {
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })));
const broken = initial.replace(/(<!-- gstack:model-policy-freshness:state:begin -->\n)[^\n]+/, '$1{broken');
const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]);
expect(readFreshnessIssue(api.issues[0].body).state).toBeNull();
expect(readFreshnessIssue(api.issues[0].body).lifecycle).toHaveLength(1);
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
expect(result.report?.status).toBe('unknown/source-unavailable');
expect(result.report?.state.lastSuccess).toBeNull();
expect(result.report?.state.lifecycle).toHaveLength(1);
expect(api.issues[0].state).toBe('open');
expect(api.issues[0].body).toContain('Human retain decision');
expect(api.issues[0].body).toContain('Human footer');
expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64'));
const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main');
expect(next.report?.state.recoveryRequired).toBe(true);
});
test('missing machine state is recovery, malformed owned boundary is not overwritten, and output is bounded', async () => {
const api = mockGitHub([{ number: 9, body: `${FRESHNESS_ISSUE_MARKER}\nMaintainer notes`, state: 'closed' }]);
const result = await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main');
expect(result.report?.status).toBe('unknown/source-unavailable');
expect(api.issues[0].state).toBe('open');
expect(api.issues[0].body).toContain('Maintainer notes');
expect(() => writeFreshnessIssue(`${FRESHNESS_ISSUE_MARKER}\n${FRESHNESS_REGION_BEGIN}\nbroken`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
expect(() => writeFreshnessIssue('x'.repeat(FRESHNESS_BOUNDS.issueBytes), assessFreshness(freshnessObservation()))).toThrow('tracking-issue-size-limit');
expect(() => writeFreshnessIssue(`${FRESHNESS_REGION_END}\n${FRESHNESS_REGION_BEGIN}`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
});
test('duplicate markers fail visibly rather than selecting an arbitrary issue', async () => {
const api = mockGitHub([{ number: 1, body: FRESHNESS_ISSUE_MARKER, state: 'open' }, { number: 2, body: FRESHNESS_ISSUE_MARKER, state: 'closed' }]);
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('duplicate-tracking-marker');
expect(api.calls).not.toContain('update');
expect(api.calls).not.toContain('create');
});
test('user-created marker copies never become authority or trigger replacement bootstrap', async () => {
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
for (const includeOwned of [false, true]) {
const api = mockGitHub([{ number: 1, body, state: 'closed', author: 'untrusted-user' }, ...(includeOwned ? [{ number: 2, body, state: 'closed' as const }] : [])]);
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('untrusted-tracking-marker-owner');
expect(api.calls).not.toContain('read');
expect(api.calls).not.toContain('update');
expect(api.calls).not.toContain('create');
}
});
test('CRLF and CR-only marker lines retain retirement history and publish to the same issue on source failure', async () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
for (const newline of ['\r\n', '\r']) {
const prefix = `Human prefix remains byte-identical.${newline}${newline}`;
const suffix = `${newline}${newline}Human suffix remains byte-identical.${newline}`;
const body = prefix + writeFreshnessIssue('', initial).replaceAll('\n', newline) + suffix;
const api = mockGitHub([{ number: 17, body, state: 'open' }]);
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
expect(result.action).toBe('updated');
expect(result.issueNumber).toBe(17);
expect(api.issues).toHaveLength(1);
expect(api.calls).not.toContain('create');
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
const stored = readFreshnessIssue(api.issues[0].body);
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
expect(stored.state?.lastSuccess).toEqual(initial.state.lastSuccess);
expect(result.report?.status).toBe('unknown/source-unavailable');
}
});
test('inline and quoted marker mentions preserve human prefix and suffix outside the real region', async () => {
const initial = assessFreshness(freshnessObservation());
const prefix = `Human note mentioning ${FRESHNESS_REGION_BEGIN} inline; preserve this entire sentence.\r\n> ${FRESHNESS_REGION_END}\r\nQuoted '${FRESHNESS_REGION_BEGIN}' and '${FRESHNESS_REGION_END}' are human prose.\r\n\r\n`;
const suffix = `\r\nHuman suffix mentions ${FRESHNESS_REGION_END} inline.\r\n> ${FRESHNESS_REGION_BEGIN}\r\n`;
const body = prefix + writeFreshnessIssue('', initial) + suffix;
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), initial.state);
const rewritten = writeFreshnessIssue(body, report);
expect(rewritten.startsWith(prefix)).toBe(true);
expect(rewritten.endsWith(suffix)).toBe(true);
expect(readFreshnessIssue(rewritten).state).toEqual(report.state);
const api = mockGitHub([{ number: 19, body, state: 'closed' }]);
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
expect(result.action).toBe('updated');
expect(api.issues).toHaveLength(1);
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
});
test('malformed or missing top markers with owned evidence never become a healthy bootstrap', async () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
for (const marker of ['', `${FRESHNESS_ISSUE_MARKER} `, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace('<!-- ', '<!--'), FRESHNESS_ISSUE_MARKER.slice(0, -1)]) {
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_ISSUE_MARKER, marker);
const api = mockGitHub([{ number: 23, body, state: 'open' }]);
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
expect(api.issues).toHaveLength(1);
expect(api.issues[0].body).toBe(body);
expect(api.calls).not.toContain('create');
expect(api.calls).not.toContain('update');
expect(readFreshnessIssue(body).lifecycle).toEqual(initial.state.lifecycle);
expect(readFreshnessIssue(body).recoveryRequired).toBe(true);
}
});
test('standalone malformed top markers without readable regions block healthy bootstrap', async () => {
for (const marker of [` ${FRESHNESS_ISSUE_MARKER}`, `${FRESHNESS_ISSUE_MARKER}\t`, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace(':v1', '-v1')]) {
const body = `Human text\r\n${marker}\r\nRetirement history needs manual recovery.`;
const api = mockGitHub([{ number: 29, body, state: 'open' }]);
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
expect(api.issues[0].body).toBe(body);
expect(api.calls).not.toContain('create');
expect(api.calls).not.toContain('update');
}
});
test('malformed owned boundaries salvage available lifecycle history but never certify a healthy publication', async () => {
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_BEGIN.replace(':begin', ':begn')).replace(FRESHNESS_REGION_END, FRESHNESS_REGION_END.replace(':end', ':en'));
const stored = readFreshnessIssue(body);
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
expect(stored.recoveryRequired).toBe(true);
const api = mockGitHub([{ number: 31, body, state: 'open' }]);
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
expect(api.issues[0].body).toBe(body);
expect(api.calls).not.toContain('create');
expect(api.calls).not.toContain('update');
});
});
describe('publication ordering and default-branch guards', () => {
test('both completion orders retain the newer unresolved retirement; old rerun is rejected even with a new observation date', async () => {
for (const newerFirst of [false, true]) {
const api = mockGitHub();
const older = freshnessObservation('100');
const newer = freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() });
for (const observation of newerFirst ? [newer, older] : [older, newer]) await publishFreshness(observation, api.github, 'refs/heads/main');
expect(api.issues[0].state).toBe('open');
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('101');
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].modelId).toBe('claude-fable-5-1');
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
rerun.run.attempt = 2;
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
}
});
test('higher run ID cannot publish an older check timestamp, but a newer attempt can advance the same run', async () => {
const api = mockGitHub();
await publishFreshness(freshnessObservation('100', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
expect((await publishFreshness(freshnessObservation('101', '2026-10-07T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
rerun.run.attempt = 2;
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('updated');
});
test('non-main is report-only with zero GitHub operations; a non-main default disables writes', async () => {
const api = mockGitHub();
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/feature')).action).toBe('report-only');
expect(api.calls).toEqual([]);
api.setIdentity({ defaultBranch: 'develop', sourceSha256: freshnessCatalog.sourceSha256 });
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('default-branch-is-not-main');
expect(api.calls).not.toContain('create');
});
test('obsolete local catalog and catalog change immediately before write are rejected', async () => {
const api = mockGitHub();
api.setIdentity({ defaultBranch: 'main', sourceSha256: 'b'.repeat(64) });
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
expect(api.calls).not.toContain('list');
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
api.github.listIssues = async () => { api.setIdentity({ defaultBranch: 'main', sourceSha256: 'c'.repeat(64) }); return []; };
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
expect(api.issues).toHaveLength(0);
});
test('final issue read preserves human text added during the authoritative catalog check', async () => {
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
const api = mockGitHub([{ number: 1, body, state: 'closed' }]);
const authority = api.github.defaultCatalog;
api.github.defaultCatalog = async signal => {
const identity = await authority(signal);
if (api.calls.includes('list')) api.issues[0].body += '\nLate maintainer note';
return identity;
};
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
expect(api.issues[0].body.endsWith('\nLate maintainer note')).toBe(true);
});
test('final issue read rejects an older observation when newer authoritative history arrives during the catalog check', async () => {
const api = mockGitHub([{ number: 1, body: writeFreshnessIssue('', assessFreshness(freshnessObservation())), state: 'closed' }]);
const authority = api.github.defaultCatalog;
api.github.defaultCatalog = async signal => {
const identity = await authority(signal);
if (api.calls.includes('list')) api.issues[0] = { ...api.issues[0], state: 'open', body: writeFreshnessIssue(api.issues[0].body, assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }))) };
return identity;
};
expect((await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
expect(api.calls).not.toContain('update');
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('102');
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].state).toBe('deprecated');
});
test('publication has a separate exact sixty-second deadline and aborts the blocked API', async () => {
const real = globalThis.setTimeout;
const installed: number[] = [];
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => { installed.push(ms); return real(callback, 5); }) as typeof setTimeout);
const api = mockGitHub();
let signal: AbortSignal | undefined;
api.github.defaultCatalog = async input => { signal = input; return new Promise(() => {}); };
try { await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('deadline-exceeded'); }
finally { timers.mockRestore(); }
expect(installed).toEqual([60_000]);
expect(signal?.aborted).toBe(true);
expect(api.issues).toHaveLength(0);
});
});
describe('network adapter, real catalog and CLI seams', () => {
test('real catalog export and canonical hash bind the four identities and separate source-file hash', () => {
const source = readFileSync(new URL('../lib/model-catalog.ts', import.meta.url), 'utf8');
const identity = freshnessCatalogIdentity(source);
expect(identity.sha256).toBe(modelCatalogSha256());
expect(identity.sourceSha256).toBe(sha256(source));
expect(identity.models).toEqual(MODEL_CATALOG.map(entry => ({ provider: entry.provider, tier: entry.tier, modelId: entry.model })));
});
test('GitHub reads all-state paginated issues, ignores PRs, and compares remote bytes without executing them', async () => {
const calls: { url: string; method?: string }[] = [];
const text = 'untrusted TS is data, never import or eval';
const api = freshnessGitHub('owner/repo', 'fixture', async (url, init) => {
calls.push({ url, method: init.method });
expect(init.redirect).toBe('error');
if (url.endsWith('/repos/owner/repo')) return Response.json({ default_branch: 'main' });
if (url.includes('/contents/')) return Response.json({ encoding: 'base64', content: Buffer.from(text).toString('base64'), size: Buffer.byteLength(text) });
if (new URL(url).searchParams.get('page') === '1') return Response.json(Array.from({ length: 100 }, (_, i) => ({ number: i + 1, body: '', state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR }, ...(i === 0 ? {} : { pull_request: {} }) })));
if (new URL(url).searchParams.get('page') === '2') return Response.json([{ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }]);
return Response.json({ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } });
});
const signal = new AbortController().signal;
expect(await api.defaultCatalog(signal)).toEqual({ defaultBranch: 'main', sourceSha256: sha256(text) });
expect((await api.listIssues(signal)).map(issue => issue.number)).toEqual([1, 101]);
await api.updateIssue(101, { body: 'machine-owned data', state: 'open' }, signal);
expect(calls.at(-1)?.method).toBe('PATCH');
expect(calls.filter(call => call.url.includes('/issues?')).every(call => call.url.includes('state=all'))).toBe(true);
expect(() => freshnessGitHub('owner/repo; shell', 'fixture')).toThrow('prerequisites-missing');
await expect(freshnessGitHub('owner/repo', 'fixture', async () => new Response(null, { status: 403 })).listIssues(signal)).rejects.toThrow('github-http-403');
});
test('creator-filtered discovery stays one request with more than one hundred unrelated issues and PRs', async () => {
const unrelated = Array.from({ length: 301 }, (_, i) => ({ number: i + 1, body: 'Unrelated public issue body', state: 'open', user: { login: 'ordinary-user' }, ...(i % 2 ? { pull_request: {} } : {}) }));
const items = [...unrelated, { number: 9000, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }];
const requests: URL[] = [];
const api = freshnessGitHub('owner/repo', 'fixture', async url => {
const request = new URL(url);
requests.push(request);
const creator = request.searchParams.get('creator');
const selected = creator ? items.filter(issue => issue.user.login === creator) : items;
const page = Number(request.searchParams.get('page'));
return Response.json(selected.slice((page - 1) * 100, page * 100));
});
const found = await api.listIssues(new AbortController().signal);
expect(requests).toHaveLength(1);
expect(requests[0].searchParams.get('creator')).toBe(FRESHNESS_ISSUE_AUTHOR);
expect(requests[0].searchParams.get('state')).toBe('all');
expect(found.map(issue => issue.number)).toEqual([9000]);
});
test('creator filter never replaces returned-owner validation', async () => {
const api = freshnessGitHub('owner/repo', 'fixture', async () => Response.json([{ number: 7, body: FRESHNESS_ISSUE_MARKER, state: 'open', user: { login: 'untrusted-user' } }]));
await expect(api.listIssues(new AbortController().signal)).rejects.toThrow('github-issue-owner-filter-mismatch');
});
test('CLI non-main --publish still has no GitHub calls; main publication failure preserves source receipts and fails honestly', async () => {
const api = mockGitHub();
const fetcher = async (url: string) => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id));
const env = { GITHUB_REF: 'refs/heads/feature', GITHUB_RUN_ID: '100', GITHUB_SHA: 'a'.repeat(40) };
const result = await runModelPolicyFreshness({ env, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
expect(result.publication?.action).toBe('report-only');
expect(api.calls).toEqual([]);
api.github.defaultCatalog = async () => { throw new Error('github-http-403'); };
const failed = await runModelPolicyFreshness({ env: { ...env, GITHUB_REF: 'refs/heads/main', GITHUB_REPOSITORY: 'owner/repo', GH_TOKEN: 'fixture' }, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
expect(failed.publication?.action).toBe('failed');
expect(failed.report.status).toBe('unknown/source-unavailable');
expect(failed.observation.sources.filter(source => source.receipt)).toHaveLength(4);
expect(api.issues).toHaveLength(0);
});
});
describe('weekly/manual workflow contract', () => {
test('one repository-wide serialized group, advisory schedule, main writes and non-main reports', () => {
const text = readFileSync(new URL('../.github/workflows/model-policy-freshness.yml', import.meta.url), 'utf8');
const workflow = Bun.YAML.parse(text) as { concurrency: { group: string; 'cancel-in-progress': boolean }; on: { schedule: { cron: string }[]; workflow_dispatch: unknown }; jobs: Record<string, { steps: { name?: string; if?: string; run?: string }[] }> };
expect(workflow.concurrency).toEqual({ group: 'model-policy-freshness', 'cancel-in-progress': false });
expect(workflow.on.schedule).toEqual([{ cron: '0 11 * * 1' }]);
expect(Object.keys(workflow.jobs)).toEqual(['advisory']);
expect(text).toContain('name: Model-policy freshness');
expect(text).toContain('workflow_dispatch:');
expect(text).toContain("cron: '0 11 * * 1'");
expect(text.match(/group:/g)).toHaveLength(1);
expect(text).toContain('group: model-policy-freshness\n cancel-in-progress: false');
expect(text).not.toContain('model-policy-freshness-${{ github.ref }}');
expect(text).toContain("if: github.ref == 'refs/heads/main'");
expect(text).toContain("if: github.ref != 'refs/heads/main'");
expect(text).toContain('run: bun scripts/model-policy-freshness.ts --publish');
expect(text).toContain('run: bun scripts/model-policy-freshness.ts\n');
expect(text).toContain('if-no-files-found: error');
expect(text).toContain('if: always()');
expect(text).toContain('bun-version: 1.4.2');
expect(text).toContain('persist-credentials: false');
expect(text).toContain('actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a');
expect(text.match(/secrets\.[A-Z_]+/g)).toEqual(['secrets.GITHUB_TOKEN']);
expect(text).not.toMatch(/pull_request|ANTHROPIC_API_KEY|OPENAI_API_KEY|codex exec|claude -p|gh issue|eval:bg|gstack-config set/);
expect(text.match(/uses: [^\n]+/g)?.every(line => /@[a-f0-9]{40}$/.test(line))).toBe(true);
});
});
+524
View File
@@ -0,0 +1,524 @@
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { catalogEntry, MODEL_CATALOG, modelCatalogCanonicalJson, modelCatalogSha256 } from '../lib/model-catalog';
import {
ANTHROPIC_MODEL_ID_PATTERN,
CUSTOM_PROVIDER_UNSCANNED,
describeSelection,
detectCustomProvider as detectWith,
isValidModelId,
ModelPolicyError,
OPENAI_MODEL_ID_PATTERN,
parseModelPolicyConfig,
readModelPolicyConfig,
resolveImplementationModels as implementationWith,
resolvePlanReviewModel as planReviewWith,
selectionRepair,
validatePolicyValue,
type ModelSelection,
} from '../lib/model-policy';
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
import { CODEX_RUNTIME_MODEL_PATTERN } from '../scripts/resolve-codex-generation-model';
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';
const ROOT = path.resolve(import.meta.dir, '..');
const roots: string[] = [];
const ISOLATED_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-native-'));
const ISOLATED = { cwd: ISOLATED_DIR, claudeManagedDir: path.join(ISOLATED_DIR, 'managed'), codexSystemConfig: null };
const resolvePlanReviewModel = (opts: Parameters<typeof planReviewWith>[0]) => planReviewWith({ ...ISOLATED, ...opts });
const resolveImplementationModels = (opts: Parameters<typeof implementationWith>[0]) => implementationWith({ ...ISOLATED, ...opts });
const detectCustomProvider = (provider: Parameters<typeof detectWith>[0], opts: Parameters<typeof detectWith>[1] = {}) => detectWith(provider, { ...ISOLATED, ...opts });
afterAll(() => fs.rmSync(ISOLATED_DIR, { recursive: true, force: true }));
afterEach(() => {
for (const root of roots.splice(0)) {
fs.chmodSync(root, 0o700);
fs.rmSync(root, { recursive: true, force: true });
}
});
function fixture(config?: string) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-'));
roots.push(root);
const state = path.join(root, 'state');
const home = path.join(root, 'home');
fs.mkdirSync(state);
fs.mkdirSync(home);
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
const env: Record<string, string | undefined> = { HOME: home, GSTACK_STATE_ROOT: state };
return { root, state, home, env };
}
function policyError(run: () => unknown): ModelPolicyError {
try {
run();
} catch (error) {
if (error instanceof ModelPolicyError) return error;
throw error;
}
throw new Error('expected a ModelPolicyError');
}
function selected(selection: ModelSelection): string {
if (selection.status !== 'selected') throw new Error(`expected selected, got ${selection.status}`);
return selection.requestedModel;
}
function tree(dir: string): string[] {
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
}
describe('model catalog', () => {
test('ships exactly the four approved tier IDs with provenance', () => {
expect(MODEL_CATALOG.map(e => [e.tier, e.provider, e.model])).toEqual([
['frontier', 'anthropic', 'claude-fable-5-1'],
['frontier', 'openai', 'gpt-6-astra'],
['smart', 'anthropic', 'claude-opus-5-5'],
['smart', 'openai', 'gpt-6.1-sol'],
]);
for (const entry of MODEL_CATALOG) {
expect(entry.sourceUrl).toMatch(/^https:\/\/(platform\.claude\.com|developers\.openai\.com)\//);
expect(entry.verifiedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/);
expect(Object.isFrozen(entry)).toBe(true);
}
expect(Object.isFrozen(MODEL_CATALOG)).toBe(true);
expect(catalogEntry('openai', 'smart').model).toBe('gpt-6.1-sol');
});
test('identity is deterministic and independent of user config', () => {
const json = modelCatalogCanonicalJson();
expect(JSON.parse(json)).toEqual({ version: 1, entries: MODEL_CATALOG.map(e => ({ ...e })) });
expect(modelCatalogSha256()).toBe(createHash('sha256').update(json).digest('hex'));
expect(modelCatalogCanonicalJson()).toBe(json);
const source = fs.readFileSync(path.join(ROOT, 'lib/model-catalog.ts'), 'utf-8');
expect(source).not.toMatch(/process\.env|state-root|readFileSync|config\.yaml/);
});
test('eval rulers and the no-role Codex pattern stay independent of the catalog', () => {
expect(CLAUDE_FRONTIER_EVAL_MODEL).toBe('claude-fable-5-1');
expect(CODEX_FRONTIER_MODEL).toBe('gpt-6-astra');
for (const file of ['lib/eval-model.ts', 'scripts/resolvers/constants.ts']) {
expect(fs.readFileSync(path.join(ROOT, file), 'utf-8')).not.toMatch(/from\s+['"][^'"]*model-(catalog|policy)/);
}
expect(OPENAI_MODEL_ID_PATTERN.source).toBe(CODEX_RUNTIME_MODEL_PATTERN.source);
});
});
describe('model ID validation', () => {
test.each([
['claude-opus-5-5', true],
['us.anthropic.claude-opus-5-5-v1:0', true],
['arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude', true],
['claude-opus-4@20250514', true],
['claude-opus-4-6[1m]', true],
['vendor+claude', true],
['a'.repeat(512), true],
['a'.repeat(513), false],
['claude opus', false],
['claude\topus', false],
['claude\u0007', false],
['$(touch x)', false],
['claude;rm', false],
['', false],
])('anthropic %p => %p', (value, ok) => {
expect(isValidModelId('anthropic', value)).toBe(ok);
expect(ANTHROPIC_MODEL_ID_PATTERN.test(value)).toBe(ok);
});
test.each([
['gpt-6.1-sol', true],
['org/model:v1', true],
['a'.repeat(100), true],
['a'.repeat(101), false],
['gpt+6', false],
['gpt 6', false],
['gpt"6', false],
])('openai %p => %p', (value, ok) => {
expect(isValidModelId('openai', value)).toBe(ok);
});
test('policy values are trimmed and empty values fail', () => {
expect(validatePolicyValue('model_smart_claude', ' claude-x ')).toBeNull();
expect(validatePolicyValue('plan_review_tier', ' host ')).toBeNull();
expect(validatePolicyValue('model_smart_claude', ' ')?.reason).toBe('empty_value');
expect(validatePolicyValue('implementation_tier', 'host')?.reason).toBe('invalid_implementation_tier');
expect(validatePolicyValue('plan_review_tier', 'fronteir')?.reason).toBe('invalid_plan_review_tier');
expect(validatePolicyValue('model_frontier_openai', 'gpt+6')?.reason).toBe('invalid_model_id');
});
});
describe('strict policy config snapshot', () => {
test('a missing file or absent keys give the documented defaults', () => {
const { env } = fixture();
const config = readModelPolicyConfig({ env });
expect(config.exists).toBe(false);
expect(config.planReviewTier).toMatchObject({ value: 'frontier', origin: 'default' });
expect(config.implementationTier).toMatchObject({ value: 'smart', origin: 'default' });
expect(config.overrides).toEqual({ anthropic: { frontier: null, smart: null }, openai: { frontier: null, smart: null } });
expect(readModelPolicyConfig({ env: fixture('telemetry: off\n').env }).exists).toBe(true);
});
test('last value wins, CRLF and comments are tolerated, values are trimmed', () => {
const config = parseModelPolicyConfig('# plan_review_tier: host\r\nplan_review_tier: host\r\nplan_review_tier: smart \r\nmodel_smart_openai: gpt-x\r\n', '/c.yaml');
expect(config.planReviewTier).toMatchObject({ value: 'smart', origin: 'config', path: '/c.yaml' });
expect(config.overrides.openai.smart).toBe('gpt-x');
});
test.each([
['plan_review_tier:\n', 'empty_value', 'plan_review_tier'],
['plan_review_tier: fronteir\n', 'invalid_plan_review_tier', 'plan_review_tier'],
['implementation_tier: host\n', 'invalid_implementation_tier', 'implementation_tier'],
['model_frontier_openai: gpt 6\n', 'invalid_model_id', 'model_frontier_openai'],
[' plan_review_tier: smart\n', 'config_malformed', 'plan_review_tier'],
['model_smart_claude : claude-x\n', 'config_malformed', 'model_smart_claude'],
])('%p fails with %p', (text, reason, key) => {
const error = policyError(() => parseModelPolicyConfig(text, '/c.yaml'));
expect(error.reason).toBe(reason as ModelPolicyError['reason']);
expect(error.key).toBe(key);
expect(error.repair.join('\n')).toContain(`gstack-config unset ${key}`);
expect(error.docs).toContain('troubleshooting.md#model-policy-config');
for (const step of error.repair) expect(step).not.toMatch(/[|]/);
});
test('an invalid unused override still fails closed', () => {
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_claude: bad value\n');
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('invalid_model_id');
});
test('a directory, an unreadable file or a file state root is an error, never the default', () => {
const dir = fixture();
fs.mkdirSync(path.join(dir.state, 'config.yaml'));
const isDir = policyError(() => readModelPolicyConfig({ env: dir.env }));
expect(isDir).toMatchObject({ reason: 'config_unreadable', cause: 'config.yaml is a directory' });
const notDir = fixture();
const fileRoot = path.join(notDir.root, 'file-root');
fs.writeFileSync(fileRoot, '');
expect(policyError(() => readModelPolicyConfig({ env: { ...notDir.env, GSTACK_STATE_ROOT: fileRoot } })).reason).toBe('config_unreadable');
if (process.platform !== 'win32' && process.getuid?.() !== 0) {
const denied = fixture('plan_review_tier: smart\n');
fs.chmodSync(path.join(denied.state, 'config.yaml'), 0o000);
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env: denied.env }))).toMatchObject({ reason: 'config_unreadable', cause: 'permission denied' });
}
});
test('a bound snapshot is used for the whole resolution even if the file is replaced', () => {
const { state, env } = fixture('model_frontier_openai: gpt-old\n');
const config = readModelPolicyConfig({ env });
const next = path.join(state, 'config.yaml.next');
fs.writeFileSync(next, 'plan_review_tier: smart\nmodel_frontier_openai: gpt-new\n');
fs.renameSync(next, path.join(state, 'config.yaml'));
expect(selected(resolvePlanReviewModel({ provider: 'openai', env, config }))).toBe('gpt-old');
const fresh = resolvePlanReviewModel({ provider: 'openai', env });
expect(fresh.tier).toBe('smart');
expect(selected(fresh)).toBe('gpt-6.1-sol');
});
});
describe('plan-review precedence', () => {
test.each(['anthropic', 'openai'] as const)('%s: request > env > per-tier config > catalog', provider => {
const envName = provider === 'anthropic' ? 'GSTACK_CLAUDE_MODEL' : 'GSTACK_CODEX_MODEL';
const key = provider === 'anthropic' ? 'model_frontier_claude' : 'model_frontier_openai';
const { env, state } = fixture(`${key}: tier-model\n`);
const withEnv = { ...env, [envName]: ' env-model ' };
const request = resolvePlanReviewModel({ provider, requestedModel: 'request-model', env: withEnv });
expect(request).toMatchObject({ status: 'selected', role: 'plan-review', provider, tier: 'frontier', requestedModel: 'request-model', source: { kind: 'request' } });
const fromEnv = resolvePlanReviewModel({ provider, env: withEnv });
expect(fromEnv).toMatchObject({ requestedModel: 'env-model', source: { kind: 'env', name: envName } });
const fromConfig = resolvePlanReviewModel({ provider, env });
expect(fromConfig).toMatchObject({ requestedModel: 'tier-model', source: { kind: 'config', key, path: path.join(state, 'config.yaml') } });
const fromCatalog = resolvePlanReviewModel({ provider, env: fixture().env });
expect(fromCatalog).toMatchObject({ requestedModel: catalogEntry(provider, 'frontier').model, source: { kind: 'catalog', catalogVersion: 1, verifiedAt: '2026-10-07' } });
expect(fromCatalog.tierFrom).toMatchObject({ key: 'plan_review_tier', origin: 'default' });
});
test('smart tier reads the smart override, not the frontier one', () => {
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_openai: gpt-frontier-pin\n');
const selection = resolvePlanReviewModel({ provider: 'openai', env });
expect(selection).toMatchObject({ tier: 'smart', requestedModel: 'gpt-6.1-sol', tierFrom: { origin: 'config' } });
});
test('host mode delegates with a null model; request and env still win; tier overrides do not apply', () => {
const { env } = fixture('plan_review_tier: host\nmodel_frontier_claude: claude-pin\nmodel_smart_claude: claude-pin\n');
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env });
expect(delegated).toMatchObject({ status: 'delegated-host', requestedModel: null, tier: 'host', catalog: null, source: { kind: 'host', reason: 'host-mode', key: 'plan_review_tier' } });
expect(describeSelection(delegated)).toContain('host-controlled');
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-env' } }))).toBe('claude-env');
expect(selected(resolvePlanReviewModel({ provider: 'openai', requestedModel: 'gpt-req', env }))).toBe('gpt-req');
});
test('an invalid environment override names the env var and its repair, not a tier', () => {
const { env } = fixture();
const error = policyError(() => resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt"; rm -rf /' } }));
expect(error).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', source: 'env' });
expect(error.repair[0]).toBe('unset GSTACK_CODEX_MODEL');
expect(error.docs).toContain('#model-policy-selection');
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', requestedModel: 'two words', env })).key).toBe('request');
});
test('an empty env override is unset, matching the legacy readers', () => {
const { env } = fixture();
expect(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: '' } }).source.kind).toBe('catalog');
});
test('selectionRepair follows the winning source', () => {
const { env } = fixture();
expect(selectionRepair(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toEqual(['unset GSTACK_CODEX_MODEL', 'export GSTACK_CODEX_MODEL=<model-id>']);
const catalogRepair = selectionRepair(resolvePlanReviewModel({ provider: 'anthropic', env }));
expect(catalogRepair).toContain('gstack-config set model_frontier_claude <model-id>');
expect(catalogRepair).toContain('gstack-config set plan_review_tier host');
});
});
describe('custom-provider guard', () => {
test.each(['CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY'])('%s blocks the public default but not explicit choices', flag => {
const { env } = fixture();
for (const value of ['1', 'true', 'YES', 'on']) {
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }));
expect(error).toMatchObject({ reason: 'custom_provider_requires_model', source: 'native' });
expect(error.cause).toContain(flag);
expect(error.repair).toEqual(['gstack-config set model_frontier_claude <model-id-your-endpoint-serves>', 'export GSTACK_CLAUDE_MODEL=<model-id-your-endpoint-serves>', 'gstack-config set plan_review_tier host']);
expect(error.docs).toContain('#model-policy-provider');
}
for (const value of ['0', 'false', '', 'no']) {
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }).source.kind).toBe('catalog');
}
const custom = { ...env, [flag]: '1' };
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...custom, GSTACK_CLAUDE_MODEL: 'us.anthropic.claude-x-v1:0' } }))).toBe('us.anthropic.claude-x-v1:0');
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('plan_review_tier: host\n').env, [flag]: '1' } }).status).toBe('delegated-host');
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('model_frontier_claude: vertex-claude@1\n').env, [flag]: '1' } }))).toBe('vertex-claude@1');
expect(resolvePlanReviewModel({ provider: 'openai', env: custom }).source.kind).toBe('catalog');
});
test('ANTHROPIC_BASE_URL counts only when non-default and is never printed', () => {
const { env } = fixture();
expect(detectCustomProvider('anthropic', { env: { ...env, ANTHROPIC_BASE_URL: 'https://api.anthropic.com/' } }).custom).toBe(false);
const endpoint = new URL('https://proxy.example/v1');
endpoint.username = 'fixture-user';
endpoint.password = 'fixture-password';
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, ANTHROPIC_BASE_URL: endpoint.href } }));
expect(error.reason).toBe('custom_provider_requires_model');
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.password);
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.username);
});
test('Claude settings.json env signals count; malformed settings are unresolved', () => {
const { env, home } = fixture();
fs.mkdirSync(path.join(home, '.claude'));
const settings = path.join(home, '.claude', 'settings.json');
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } }));
expect(detectCustomProvider('anthropic', { env }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${settings})`]);
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '0' }, model: 'opus' }));
expect(detectCustomProvider('anthropic', { env }).custom).toBe(false);
fs.writeFileSync(settings, '{not json');
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env }))).toMatchObject({ reason: 'native_provider_unresolved', key: settings });
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-x' } }))).toBe('claude-x');
const other = path.join(home, 'alt');
fs.mkdirSync(other);
fs.writeFileSync(path.join(other, 'settings.json'), JSON.stringify({ env: { CLAUDE_CODE_USE_VERTEX: 'true' } }));
expect(detectCustomProvider('anthropic', { env: { ...env, CLAUDE_CONFIG_DIR: other } }).custom).toBe(true);
});
test('Codex model_provider, profile inheritance and base URLs', () => {
const { env, home } = fixture();
const codex = path.join(home, '.codex');
fs.mkdirSync(codex);
const write = (toml: string) => fs.writeFileSync(path.join(codex, 'config.toml'), toml);
const custom = () => detectCustomProvider('openai', { env }).custom;
expect(custom()).toBe(false);
write('model = "gpt-5.5"\nmodel_provider = "openai"\n');
expect(custom()).toBe(false);
write('model_provider = "azure"\n');
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('custom_provider_requires_model');
write('model_provider = "azure"\nprofile = "work"\n[profiles.work]\nmodel_provider = "openai"\n');
expect(custom()).toBe(false);
write('profile = "work"\n[profiles.work]\nmodel_provider = "ollama"\n');
expect(custom()).toBe(true);
write('profile = "missing"\n');
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
write('model_provider = 7\n');
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
write('model_provider = [\n');
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('native_provider_unresolved');
expect(selected(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toBe('gpt-x');
write('openai_base_url = "https://gateway.example/v1"\n');
expect(custom()).toBe(true);
write('openai_base_url = "https://api.openai.com/v1/"\n');
expect(custom()).toBe(false);
expect(detectCustomProvider('openai', { env: { ...env, OPENAI_BASE_URL: 'https://gateway.example/v1' } }).custom).toBe(true);
expect(policyError(() => detectCustomProvider('openai', { env: { ...env, CODEX_HOME: 'relative/codex' } })).reason).toBe('native_provider_unresolved');
const alt = path.join(home, 'codex home with spaces');
fs.mkdirSync(alt);
fs.writeFileSync(path.join(alt, 'config.toml'), 'model_provider = "azure"\n');
expect(detectCustomProvider('openai', { env: { ...env, CODEX_HOME: alt } }).custom).toBe(true);
});
});
describe('custom-provider settings scopes', () => {
function project(settings: Record<string, string>) {
const fx = fixture();
for (const [rel, body] of Object.entries(settings)) {
fs.mkdirSync(path.dirname(path.join(fx.root, rel)), { recursive: true });
fs.writeFileSync(path.join(fx.root, rel), body);
}
return fx;
}
const proxy = JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } });
const bedrock = JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } });
test.each([
['shared project settings', 'app/.claude/settings.json', 'app'],
['project local settings', 'app/.claude/settings.local.json', 'app'],
['repository-root local settings from a subdirectory', 'repo/.claude/settings.local.json', 'repo/pkg'],
])('%s block the public default', (_name, file, cwd) => {
const fx = project({ [file]: proxy, 'repo/.git/HEAD': 'ref: refs/heads/main\n' });
fs.mkdirSync(path.join(fx.root, cwd), { recursive: true });
const scope = { env: fx.env, cwd: path.join(fx.root, cwd) };
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', ...scope }));
expect(error.reason).toBe('custom_provider_requires_model');
expect(error.cause).toContain(path.join(fx.root, file));
expect(error.cause).not.toContain('proxy.example.test');
expect(policyError(() => resolveImplementationModels({ host: 'claude-code', ...scope })).reason).toBe('custom_provider_requires_model');
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_CLAUDE_MODEL: 'proxy-claude' } }))).toBe('proxy-claude');
expect(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_STATE_ROOT: project({ 'state/config.yaml': 'plan_review_tier: host\n' }).state } }).status).toBe('delegated-host');
expect(resolvePlanReviewModel({ provider: 'openai', ...scope }).source.kind).toBe('catalog');
});
test('a worktree also reads the main checkout local settings', () => {
const fx = project({
'main/.git/worktrees/wt/commondir': '../..\n',
'main/.claude/settings.local.json': bedrock,
});
fs.mkdirSync(path.join(fx.root, 'wt'));
fs.writeFileSync(path.join(fx.root, 'wt', '.git'), `gitdir: ${path.join(fx.root, 'main/.git/worktrees/wt')}\n`);
const detection = detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'wt') });
expect(detection.signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(fx.root, 'main/.claude/settings.local.json')})`]);
});
test('shared settings are read from the working directory, as Claude does, not the repository root', () => {
const fx = project({ 'repo/.git/HEAD': '', 'repo/.claude/settings.json': proxy });
fs.mkdirSync(path.join(fx.root, 'repo/pkg'));
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo/pkg') }).custom).toBe(false);
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo') }).custom).toBe(true);
});
test('managed settings file and drop-ins count; a malformed one is unresolved', () => {
const fx = project({ 'managed/managed-settings.d/20-route.json': bedrock, 'managed/managed-settings.d/.hidden.json': '{', 'managed/managed-settings.d/notes.txt': '{' });
const managed = path.join(fx.root, 'managed');
expect(detectCustomProvider('anthropic', { env: fx.env, claudeManagedDir: managed }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(managed, 'managed-settings.d/20-route.json')})`]);
fs.writeFileSync(path.join(managed, 'managed-settings.json'), '{broken');
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: fx.env, claudeManagedDir: managed }))).toMatchObject({ reason: 'native_provider_unresolved', key: path.join(managed, 'managed-settings.json') });
});
test('a requested cwd is honored instead of process.cwd()', () => {
const fx = project({ 'elsewhere/.claude/settings.local.json': proxy });
const cwd = path.join(fx.root, 'elsewhere');
expect(process.cwd()).not.toBe(cwd);
const managed = { claudeManagedDir: path.join(fx.root, 'none') };
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd, ...managed })).reason).toBe('custom_provider_requires_model');
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd: path.relative(process.cwd(), cwd), ...managed })).reason).toBe('custom_provider_requires_model');
const original = process.cwd();
process.chdir(fx.home);
try {
expect(planReviewWith({ provider: 'anthropic', env: fx.env, ...managed }).source.kind).toBe('catalog');
process.chdir(cwd);
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, ...managed })).reason).toBe('custom_provider_requires_model');
} finally {
process.chdir(original);
}
});
test('Codex system config is a lower layer than user config', () => {
const fx = project({ 'etc/codex.toml': 'model_provider = "azure"\n' });
const scope = { env: fx.env, codexSystemConfig: path.join(fx.root, 'etc/codex.toml') };
expect(detectCustomProvider('openai', scope).signals).toEqual([`model_provider "azure" (${scope.codexSystemConfig})`]);
fs.mkdirSync(path.join(fx.home, '.codex'));
fs.writeFileSync(path.join(fx.home, '.codex', 'config.toml'), 'model_provider = "openai"\n');
expect(detectCustomProvider('openai', scope).custom).toBe(false);
});
test('a malformed native config reached from smart plan-review or implementation names no tier override', () => {
const fx = project({ 'state/config.yaml': 'plan_review_tier: smart\n', 'home/.codex/config.toml': 'model_provider = [\n' });
for (const run of [
() => resolvePlanReviewModel({ provider: 'openai', env: fx.env }),
() => resolveImplementationModels({ provider: 'openai', env: fx.env }),
]) {
const error = policyError(run);
expect(error.reason).toBe('native_provider_unresolved');
expect(error.repair).toEqual([`fix the TOML in ${path.join(fx.home, '.codex', 'config.toml')}`, 'or name the model explicitly for this request']);
}
});
test('detection reports what it scanned and what it cannot see', () => {
const fx = fixture();
const detection = detectCustomProvider('anthropic', { env: fx.env });
expect(detection.scanned).toContain(path.join(ISOLATED_DIR, '.claude', 'settings.local.json'));
expect(detection.scanned).toContain(path.join(ISOLATED.claudeManagedDir, 'managed-settings.json'));
expect(detection.unscanned).toEqual([...CUSTOM_PROVIDER_UNSCANNED.anthropic]);
expect(detectCustomProvider('openai', { env: fx.env }).unscanned.join(' ')).toContain('--profile');
});
});
describe('implementation recommendation', () => {
test('known hosts recommend their own provider; unknown hosts get both', () => {
const { env } = fixture();
expect(resolveImplementationModels({ host: 'claude-code', env }).map(s => [s.provider, selected(s)])).toEqual([['anthropic', 'claude-opus-5-5']]);
expect(resolveImplementationModels({ host: 'codex', env }).map(s => [s.provider, selected(s)])).toEqual([['openai', 'gpt-6.1-sol']]);
for (const host of ['unknown', undefined] as const) {
expect(resolveImplementationModels({ host, env }).map(s => s.provider)).toEqual(['anthropic', 'openai']);
}
expect(resolveImplementationModels({ host: 'codex', provider: 'anthropic', env }).map(s => s.provider)).toEqual(['anthropic']);
});
test('frontier is honored, overrides apply, env overrides do not', () => {
const { env } = fixture('implementation_tier: frontier\nmodel_frontier_openai: gpt-pin\n');
const envWithOverrides = { ...env, GSTACK_CLAUDE_MODEL: 'claude-env', GSTACK_CODEX_MODEL: 'gpt-env' };
const [anthropic, openai] = resolveImplementationModels({ env: envWithOverrides });
expect(anthropic).toMatchObject({ role: 'implementation', tier: 'frontier', requestedModel: 'claude-fable-5-1', source: { kind: 'catalog' }, tierFrom: { key: 'implementation_tier', origin: 'config' } });
expect(openai).toMatchObject({ requestedModel: 'gpt-pin', source: { kind: 'config', key: 'model_frontier_openai' } });
expect(resolveImplementationModels({ provider: 'openai', requestedModel: 'gpt-req', env })[0]).toMatchObject({ requestedModel: 'gpt-req', source: { kind: 'request' } });
});
test('an explicit model without a single provider is rejected', () => {
const { env } = fixture();
expect(policyError(() => resolveImplementationModels({ requestedModel: 'gpt-x', env })).reason).toBe('provider_required');
});
test('custom routing needs an explicit implementation model; it is never delegated', () => {
const { env } = fixture();
const custom = { ...env, CLAUDE_CODE_USE_BEDROCK: '1' };
const error = policyError(() => resolveImplementationModels({ host: 'claude-code', env: custom }));
expect(error).toMatchObject({ reason: 'custom_provider_requires_model' });
expect(error.repair[0]).toBe('gstack-config set model_smart_claude <model-id-your-endpoint-serves>');
expect(error.repair.join(' ')).not.toContain('GSTACK_CLAUDE_MODEL');
expect(error.repair.join(' ')).not.toContain('host');
expect(selected(resolveImplementationModels({ host: 'claude-code', env: { ...fixture('model_smart_claude: us.anthropic.claude-x\n').env, CLAUDE_CODE_USE_BEDROCK: '1' } })[0])).toBe('us.anthropic.claude-x');
});
});
describe('purity', () => {
test('resolution and inspection never write state', () => {
const { root, env } = fixture('plan_review_tier: smart\n');
const before = tree(root);
resolvePlanReviewModel({ provider: 'openai', env });
resolvePlanReviewModel({ provider: 'anthropic', env });
resolveImplementationModels({ env });
expect(tree(root)).toEqual(before);
});
test('errors carry every structured field', () => {
const error = policyError(() => parseModelPolicyConfig('plan_review_tier: nope\n', '/c.yaml'));
expect(Object.keys(error.toJSON()).sort()).toEqual(['cause', 'docs', 'key', 'problem', 'reason', 'repair', 'source']);
expect(error.repair.length).toBeGreaterThan(0);
});
});
+485
View File
@@ -0,0 +1,485 @@
/**
* Role-bearing outside invocations: exact plan-review membership, one bound
* selection through probe and dispatch, the invocation-boundary notice, and
* no-role negative controls. Fake CLIs only; no paid calls.
*/
import { afterAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { outsideVoiceCommand, outsideVoicePreflight } from '../scripts/resolvers/outside-voice';
import { RESOLVERS } from '../scripts/resolvers/index';
import { generateAdversarialStep, generateCodexDocReview, generateCodexPlanReview, generateCodexSecondOpinion } from '../scripts/resolvers/outside-voice-steps';
import { generateDesignOutsideVoices } from '../scripts/resolvers/design';
import { generateImplementationModelHandoff } from '../scripts/resolvers/plan-review';
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
import { claudeCodeArgs } from '../lib/claude-code';
import { resolvePlanReviewModel } from '../lib/model-policy';
const ROOT = path.resolve(import.meta.dir, '..');
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'role-invocation-'));
const REPO = path.join(TMP, 'repo');
const BIN = path.join(TMP, 'bin');
const CALLS = path.join(TMP, 'calls.jsonl');
const PROMPT = path.join(TMP, 'prompt.txt');
const FAKE = path.join(TMP, 'fake.ts');
for (const dir of [REPO, BIN]) fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(PROMPT, 'Review this plan.\n');
fs.writeFileSync(FAKE, `
import { appendFileSync, writeFileSync } from 'node:fs';
const args = process.argv.slice(2);
if (args[0] === 'sandbox') process.exit(0);
if (args[0] === '--version') { console.log('codex-cli ' + (process.env.FAKE_CODEX_VERSION ?? '0.160.0')); process.exit(0); }
const claude = process.env.FAKE_AS === 'claude';
const probe = !claude && args.includes('reply OK');
appendFileSync(process.env.CALLS!, JSON.stringify({ args, probe }) + '\\n');
if (probe) {
if (process.env.REPLACE_CONFIG) writeFileSync(process.env.GSTACK_HOME + '/config.yaml', process.env.REPLACE_CONFIG);
if (process.env.FAKE_PROBE === 'model400') { console.error('ERROR: {"status":400,"error":{"message":"The model is not supported."}}'); process.exit(1); }
if (process.env.FAKE_PROBE === 'broken') { console.error('Error: spawn codex-vendor ENOENT'); process.exit(1); }
console.log('OK');
process.exit(0);
}
if (!claude || args.includes('-')) await Bun.stdin.text();
const response = 'Recommendation: split the migration because the plan couples two rollouts.';
if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } }));
else writeFileSync(args[args.indexOf('-o') + 1], response);
`);
fs.writeFileSync(path.join(BIN, 'codex'), `#!/usr/bin/env bash\nexec bun "${FAKE}" "$@"\n`, { mode: 0o755 });
const git = (args: string[]) => {
const r = spawnSync('git', args, { cwd: REPO, encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@example.invalid', GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@example.invalid' } });
if (r.status !== 0) throw new Error(r.stderr);
};
git(['init', '-q', '-b', 'main']);
fs.writeFileSync(path.join(REPO, 'plan.md'), 'plan\n');
git(['add', 'plan.md']);
git(['commit', '-qm', 'init']);
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
type Host = 'claude' | 'codex';
const ctxFor = (skillName: string, host: Host, paths = HOST_PATHS.codex): TemplateContext =>
({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, host, paths });
let counter = 0;
interface Run { stdout: string; stderr: string; status: number | null; calls: Array<{ args: string[]; probe: boolean }>; state: string }
function home(config = '', codexToml = 'model = "gpt-5.6-terra"\n') {
const dir = path.join(TMP, `h${++counter}`);
const state = path.join(dir, 'state');
fs.mkdirSync(path.join(dir, '.codex'), { recursive: true });
fs.mkdirSync(state, { recursive: true });
fs.writeFileSync(path.join(dir, '.codex', 'config.toml'), codexToml);
if (config) fs.writeFileSync(path.join(state, 'config.yaml'), config);
return { dir, state };
}
function env(h: { dir: string; state: string }, host: Host, extra: Record<string, string> = {}): Record<string, string> {
return {
PATH: `${BIN}${path.delimiter}${process.env.PATH ?? ''}`, HOME: h.dir, CODEX_HOME: path.join(h.dir, '.codex'),
GSTACK_HOME: h.state, GSTACK_ROOT: ROOT, GSTACK_BIN: path.join(ROOT, 'bin'), CALLS, CODEX_API_KEY: 'test-key',
GSTACK_CLAUDE_BIN: process.execPath, GSTACK_CLAUDE_BIN_ARGS: JSON.stringify([FAKE]), FAKE_AS: host === 'codex' ? 'claude' : 'codex',
...(host === 'codex' ? { CODEX_THREAD_ID: 'fixture', GSTACK_ACTIVE_HOST: 'codex' } : { CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' }),
...extra,
};
}
function run(host: Host, h: { dir: string; state: string }, extra: Record<string, string> = {}, role: 'plan-review' | null = 'plan-review'): Run {
fs.rmSync(CALLS, { force: true });
const command = outsideVoiceCommand(ctxFor('plan-eng-review', host), { promptFile: PROMPT, timeoutMs: 8000, ...(role ? { role } : {}) });
const r = spawnSync('bash', ['-c', command], { cwd: REPO, env: env(h, host, extra), encoding: 'utf8', timeout: 30000 });
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status, calls, state: h.state };
}
const codexModel = (call: { args: string[] }) => call.args.find(arg => arg.startsWith('model='));
const claudeModels = (call: { args: string[] }) => call.args.flatMap((arg, i) => (arg === '--model' ? [call.args[i + 1]] : arg.startsWith('--model=') ? [arg.slice(8)] : []));
const NOTICE = 'NOTICE: gstack plan reviews now use an independent plan-review model';
const marker = (state: string) => path.join(state, '.model-policy-notice-v1');
describe('plan-review role membership in shared generators', () => {
const roleBearing: Array<[string, (ctx: TemplateContext) => string]> = [
['autoplan', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan'])],
['spec', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['spec'])],
['plan-ceo-review', generateCodexPlanReview],
['plan-eng-review', generateCodexPlanReview],
['plan-devex-review', generateCodexPlanReview],
['plan-design-review', generateDesignOutsideVoices],
];
const noRole: Array<[string, (ctx: TemplateContext) => string]> = [
['design-review', generateDesignOutsideVoices],
['design-consultation', generateDesignOutsideVoices],
['office-hours', generateCodexSecondOpinion],
['office-hours', ctx => RESOLVERS.DESIGN_SKETCH(ctx)],
['review', generateAdversarialStep],
['ship', generateAdversarialStep],
['ship', ctx => RESOLVERS.DESIGN_REVIEW_LITE(ctx)],
['document-release', generateCodexDocReview],
];
for (const host of ['claude', 'codex'] as const) {
const roleMarker = host === 'claude' ? '_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?' : '--role plan-review';
for (const [skill, render] of roleBearing) {
test(`${host}: ${skill} invokes with the plan-review role only`, () => {
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
expect(text).toContain(roleMarker);
expect(text).not.toMatch(/_gstack_codex_select_model (exec|review) \|\| exit 1/);
expect(text).not.toContain('_gstack_codex_model_probe');
expect(text).not.toContain('_CODEX_MP');
expect(text).not.toContain('without overriding either');
});
}
for (const [skill, render] of noRole) {
test(`${host}: ${skill} stays no-role`, () => {
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
expect(text).not.toContain('plan-review');
expect(text).not.toContain('--role');
});
}
}
test('spec keeps medium effort, its 120s deadline and spec gate on both providers', () => {
const codex = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'claude'), ['spec']);
expect(codex).toContain(`model_reasoning_effort="medium"`);
expect(codex).toContain('_gstack_codex_timeout_wrapper 120 codex exec');
expect(codex).toMatch(/ spec "\$_OUTSIDE_TMP\/text"/);
const claude = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'codex'), ['spec']);
expect(claude).toContain('--timeout-ms 120000 --role plan-review');
});
test('role-bearing availability checks pay for no model probe; no-role checks keep theirs', () => {
for (const host of ['claude', 'codex'] as const) {
for (const text of [
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('autoplan', host, HOST_PATHS.claude), ['autoplan']),
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', host, HOST_PATHS.claude), ['opt-in', 'plan-review']),
outsideVoicePreflight(ctxFor('plan-eng-review', host, HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' }),
]) {
expect(text).not.toMatch(/_gstack_codex_(model|auth)_probe|_CODEX_MP|without overriding either/);
expect(text).toContain('the plan-review model (policy-selected, printed with its source) are checked by the actual invocation');
}
}
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).toContain('without overriding either');
expect(generateCodexDocReview(ctxFor('document-release', 'claude', HOST_PATHS.claude))).toContain('{ _gstack_codex_model_probe; _CODEX_MP=$?; }');
expect(generateAdversarialStep(ctxFor('review', 'claude', HOST_PATHS.claude))).toContain('_gstack_codex_model_probe review');
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).not.toContain('_gstack_codex_model_probe');
});
});
describe('Codex plan-review invocation binds one selection', () => {
test('a known-bad CLI version still warns without changing its advisory policy', () => {
const result = run('claude', home(), { FAKE_CODEX_VERSION: '0.120.2' });
expect(result.status).toBe(0);
expect(result.stdout).toMatch(/WARN:.*0\.120\.2.*known stdin deadlock/);
expect(result.calls.filter(call => call.probe)).toHaveLength(1);
expect(result.calls.filter(call => !call.probe)).toHaveLength(1);
expect(result.calls.every(call => codexModel(call) === 'model="gpt-6-astra"')).toBe(true);
});
test('frontier catalog model wins over a native pin; probe and dispatch use it; notice once', () => {
const h = home();
const first = run('claude', h);
expect(first.status).toBe(0);
expect(first.stdout).toContain('OUTSIDE_STATUS: completed provider=codex');
expect(first.calls.map(c => c.probe)).toEqual([true, false]);
expect(first.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
expect(first.stderr).toContain('CODEX_MODEL: gpt-6-astra (exec; role: plan-review, tier: frontier; source: gstack catalog frontier/openai');
expect(first.stderr).toContain(NOTICE);
expect(first.stderr).toContain('gstack-config set plan_review_tier smart');
expect(first.stderr.indexOf(NOTICE)).toBeLessThan(first.stderr.indexOf('CODEX_MODEL:'));
expect(fs.existsSync(marker(h.state))).toBe(true);
const second = run('claude', h);
expect(second.status).toBe(0);
expect(second.stderr).not.toContain('NOTICE: gstack plan reviews');
expect(second.stdout).toContain('MODEL_OK (cached)');
expect(second.calls.map(c => c.probe)).toEqual([false]);
});
test('negative control: the same call without a role keeps native selection, no probe, no notice', () => {
const h = home();
const r = run('claude', h, {}, null);
expect(r.status).toBe(0);
expect(r.calls.map(c => c.probe)).toEqual([false]);
expect(r.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
expect(fs.existsSync(marker(h.state))).toBe(false);
});
test('a config replaced between probe and dispatch affects only the next invocation', () => {
const h = home();
const r = run('claude', h, { REPLACE_CONFIG: 'model_frontier_openai: gpt-replaced\n' });
expect(r.status).toBe(0);
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
const next = run('claude', h);
expect(next.calls.map(codexModel)).toEqual(['model="gpt-replaced"', 'model="gpt-replaced"']);
expect(next.stderr).toContain('source: model_frontier_openai in');
});
for (const [label, config, extra, model, fragment] of [
['smart tier', 'plan_review_tier: smart\n', {}, 'gpt-6.1-sol', 'tier: smart'],
['env override', '', { GSTACK_CODEX_MODEL: 'gpt-env-pick' }, 'gpt-env-pick', 'source: GSTACK_CODEX_MODEL'],
['host mode', 'plan_review_tier: host\n', {}, 'gpt-5.6-terra', 'source: plan_review_tier=host'],
] as const) {
test(`${label} reaches both probe and dispatch with its real source`, () => {
const r = run('claude', home(config), extra);
expect(r.status).toBe(0);
expect(r.calls.map(codexModel)).toEqual([`model="${model}"`, `model="${model}"`]);
expect(r.stderr).toContain(fragment);
});
}
test('invalid policy config and unrouted custom providers stop before any Codex process', () => {
const invalid = run('claude', home('plan_review_tier: turbo\n'));
expect(invalid.status).toBe(1);
expect(invalid.calls).toEqual([]);
expect(invalid.stderr).toContain('gstack-config unset plan_review_tier');
expect(invalid.stderr).toContain('outside review unavailable; missing coverage');
const custom = run('claude', home('', 'model_provider = "azure"\n'));
expect(custom.status).toBe(1);
expect(custom.calls).toEqual([]);
expect(custom.stderr).toContain('model_frontier_openai');
expect(custom.stderr).not.toContain('NOTICE: gstack plan reviews');
});
test('a rejected role model is unusable with a source-aware repair and never dispatches', () => {
const r = run('claude', home(), { FAKE_PROBE: 'model400' });
expect(r.status).toBe(1);
expect(r.calls.map(c => c.probe)).toEqual([true]);
expect(r.stdout).toContain('MODEL_UNUSABLE');
expect(r.stdout).toContain('source: gstack catalog frontier/openai');
expect(r.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
expect(r.stdout).not.toContain('set model in');
});
test('a broken CLI found by the role probe exits 2 and never dispatches', () => {
const r = run('claude', home(), { FAKE_PROBE: 'broken' });
expect(r.status).toBe(2);
expect(r.calls.map(c => c.probe)).toEqual([true]);
expect(r.stdout).toContain('MODEL_UNUSABLE_INSTALL');
expect(r.stdout).toContain('npm install -g @openai/codex');
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
});
test('missing authentication makes no Codex call', () => {
const r = run('claude', home(), { CODEX_API_KEY: '' });
expect(r.status).toBe(1);
expect(r.calls).toEqual([]);
expect(r.stdout).toContain('AUTH_FAILED');
});
test('an explicit request outranks GSTACK_CODEX_MODEL in the probe helper', () => {
const h = home();
const r = spawnSync('bash', ['-c', `source "${ROOT}/bin/gstack-codex-probe" && _gstack_codex_select_model exec gpt-request plan-review && echo "SEL=$_GSTACK_CODEX_SEL"`],
{ env: env(h, 'claude', { GSTACK_CODEX_MODEL: 'gpt-env' }), encoding: 'utf8', timeout: 15000 });
expect(r.stdout).toContain('SEL=gpt-request');
expect(r.stderr).toContain('source: explicit request');
});
});
describe('Claude plan-review invocation emits one selected model or none', () => {
const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!);
test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => {
const h = home();
const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' });
expect(r.status).toBe(0);
expect(r.calls).toHaveLength(1);
expect(claudeModels(r.calls[0])).toEqual(['claude-fable-5-1']);
expect(result(r).selection).toMatchObject({ role: 'plan-review', tier: 'frontier', status: 'selected', requested_model: 'claude-fable-5-1' });
expect(result(r).modelUsage).toHaveProperty('model-actual');
expect(r.stderr).toContain('CLAUDE_MODEL: plan-review via anthropic: claude-fable-5-1');
expect(r.stderr).toContain(NOTICE);
expect(run('codex', h).stderr).not.toContain('NOTICE: gstack plan reviews');
});
test('GSTACK_CLAUDE_MODEL ranks inside the record and appears once', () => {
const r = run('codex', home(), { GSTACK_CLAUDE_MODEL: 'claude-env-pick' });
expect(claudeModels(r.calls[0])).toEqual(['claude-env-pick']);
expect(result(r).selection.source).toContain('GSTACK_CLAUDE_MODEL');
});
test('host mode delegates: zero --model flags and an unknown requested model', () => {
const r = run('codex', home('plan_review_tier: host\n'));
expect(r.status).toBe(0);
expect(claudeModels(r.calls[0])).toEqual([]);
expect(result(r).selection).toMatchObject({ status: 'delegated-host', requested_model: null });
});
test('custom Anthropic routing without an explicit model never spawns Claude', () => {
const r = run('codex', home(), { ANTHROPIC_BASE_URL: 'https://proxy.example.invalid' });
expect(r.status).not.toBe(0);
expect(r.calls).toEqual([]);
expect(r.stdout).toContain('custom_provider_requires_model');
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
});
test('target-repo project settings with a custom endpoint stop a catalog default, whatever the parent cwd', () => {
const target = path.join(TMP, 'custom-target');
fs.mkdirSync(path.join(target, '.claude'), { recursive: true });
fs.writeFileSync(path.join(target, '.claude', 'settings.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://gateway.example.invalid' } }));
const sub = path.join(target, 'pkg');
fs.mkdirSync(sub, { recursive: true });
spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: target, timeout: 5000 });
const h = home();
fs.rmSync(CALLS, { force: true });
const direct = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
expect(direct.status).toBe(1);
expect(JSON.parse(direct.stdout).error.code).toBe('custom_provider_requires_model');
expect(fs.existsSync(CALLS)).toBe(false);
const command = outsideVoiceCommand(ctxFor('plan-eng-review', 'codex'), { promptFile: PROMPT, timeoutMs: 5000, role: 'plan-review' });
const generated = spawnSync('bash', ['-c', command], { cwd: sub, env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
expect(generated.status).not.toBe(0);
expect(generated.stdout).toContain('custom_provider_requires_model');
expect(fs.existsSync(CALLS)).toBe(false);
const explicit = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex', { GSTACK_CLAUDE_MODEL: 'gateway-model' }), encoding: 'utf8', timeout: 20000 });
expect(explicit.status).toBe(0);
expect(JSON.parse(explicit.stdout).selection.requested_model).toBe('gateway-model');
});
test('negative control: no role keeps zero --model without an override and shows no notice', () => {
const h = home();
const r = run('codex', h, {}, null);
expect(r.status).toBe(0);
expect(claudeModels(r.calls[0])).toEqual([]);
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
expect(fs.existsSync(marker(h.state))).toBe(false);
});
test('a selected record is never re-resolved from the environment', () => {
const cmd = { command: 'claude', argsPrefix: [] };
const h = home();
const selected = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
if (selected.status !== 'selected') throw new Error('expected a selected record');
const args = claudeCodeArgs({ access: 'none', selection: selected }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' });
expect(claudeModels({ args })).toEqual(['claude-fable-5-1']);
const dashed = claudeCodeArgs({ access: 'none', selection: { ...selected, requestedModel: '-p' } }, cmd, {});
expect(dashed.filter(arg => arg === '-p')).toHaveLength(1);
expect(dashed).toContain('--model=-p');
fs.writeFileSync(path.join(h.state, 'config.yaml'), 'plan_review_tier: host\n');
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
expect(claudeModels({ args: claudeCodeArgs({ access: 'none', selection: delegated }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' }) })).toEqual([]);
expect(claudeCodeArgs({ access: 'none' }, cmd, { GSTACK_CLAUDE_MODEL: 'legacy pick' }).slice(-2)).toEqual(['--model', 'legacy pick']);
});
});
describe('manual /codex entry: explicit role only, paid probe only for the dispatched model', () => {
const skill = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md.tmpl'), 'utf8');
const preflight = skill.match(/```bash\n(_TEL=[^\n]*\n_CODEX_ROLE=''[\s\S]*?)\n```/)![1]!
.replace('{{OUTSIDE_SELF_GUARD:codex}}', RESOLVERS.OUTSIDE_SELF_GUARD(ctxFor('codex', 'claude', HOST_PATHS.claude), ['codex']));
const withRole = (text: string, role: string) => text.replaceAll("_CODEX_ROLE=''", `_CODEX_ROLE='${role}'`);
const shell = (h: { dir: string; state: string }, script: string) => {
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
if (!fs.existsSync(path.join(h.dir, '.claude', 'skills', 'gstack'))) fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
fs.rmSync(CALLS, { force: true });
const r = spawnSync('bash', ['-c', script], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 30000 });
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
return { ...r, calls: calls.filter((c: { args: string[] }) => c.args[0] === 'exec') };
};
test('Step 0.5 probes the native model without a role and makes no model call with it', () => {
const plain = shell(home(), preflight);
expect(plain.status).toBe(0);
expect(plain.stdout).toContain('MODEL_OK');
expect(plain.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
const role = shell(home(), withRole(preflight, 'plan-review'));
expect(role.status).toBe(0);
expect(role.stdout).not.toMatch(/MODEL_OK|MODEL_UNUSABLE|AUTH_FAILED/);
expect(role.calls).toEqual([]);
expect(role.stderr).not.toContain('NOTICE: gstack plan reviews');
});
for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) {
const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8');
const selections = [...template.matchAll(/_CODEX_ROLE=''\n(_gstack_codex_select_model (exec|review) '' "\$_CODEX_ROLE" "\$_REPO_ROOT" \|\| exit 1\n\[ -z "\$_CODEX_ROLE" \] \|\| _gstack_codex_model_probe \2 \|\| exit \$\?)\n/g)];
test(`${file}: every dispatch block selects then, with the role only, probes that same model`, () => {
expect(selections.length).toBe(file === 'review-mode' ? 2 : 1);
expect(template.match(/_gstack_codex_select_model/g)!.length).toBe(selections.length + (file === 'review-mode' ? 1 : 0));
for (const [block, , kind] of selections) {
const script = (role: string) => `_REPO_ROOT='${REPO}'\nsource "${ROOT}/bin/gstack-codex-probe" || exit 1\n${withRole(block, role)}\necho "DISPATCH=$_GSTACK_CODEX_SEL"`;
const plain = shell(home(), script(''));
expect(plain.status).toBe(0);
expect(plain.stdout).toContain('DISPATCH=gpt-5.6-terra');
expect(plain.calls).toEqual([]);
const h = home();
const role = shell(h, script('plan-review'));
expect(role.status).toBe(0);
expect(role.stdout).toContain('DISPATCH=gpt-6-astra');
expect(role.calls.map(codexModel)).toEqual(['model="gpt-6-astra"']);
expect(role.stderr).toContain(`CODEX_MODEL: gpt-6-astra (${kind}; role: plan-review, tier: frontier`);
expect(role.stderr).toContain(NOTICE);
const rejected = shell(home(), `export FAKE_PROBE=model400\n${script('plan-review')}`);
expect(rejected.status).toBe(1);
expect(rejected.stdout).not.toContain('DISPATCH=');
expect(rejected.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
}
});
}
});
describe('invocation-boundary migration notice', () => {
test('pure inspection writes nothing and does not suppress the first real notice', () => {
const h = home();
const inspect = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'plan-review', '--provider', 'openai', '--json'],
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
expect(inspect.status).toBe(0);
expect(fs.readdirSync(h.state)).toEqual([]);
expect(run('claude', h).stderr).toContain(NOTICE);
});
for (const host of ['codex', 'claude'] as const) {
test(`${host === 'codex' ? 'Claude' : 'Codex'}: an unwritable marker repeats the notice instead of hiding it`, () => {
const h = home();
fs.chmodSync(h.state, 0o500);
try {
for (let i = 0; i < 2; i++) {
const r = run(host, h);
expect(r.status).toBe(0);
expect(r.stderr).toContain(NOTICE);
}
expect(fs.existsSync(marker(h.state))).toBe(false);
} finally { fs.chmodSync(h.state, 0o700); }
});
}
test('disabled reviews stay disabled: no probe, no notice, no Codex call', () => {
const block = outsideVoicePreflight(ctxFor('plan-eng-review', 'claude', HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' })
.match(/```bash\n([\s\S]*?)\n```/)![1]!;
for (const [config, mode] of [['codex_reviews: disabled\n', 'CODEX_MODE: disabled'], ['', 'CODEX_MODE: ready']] as const) {
const h = home(config);
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
fs.rmSync(CALLS, { force: true });
const r = spawnSync('bash', ['-c', block], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 20000 });
expect(r.stdout).toContain(mode);
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8') : '';
expect(calls).not.toContain('"exec"');
if (config) expect(calls).toBe('');
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
expect(fs.existsSync(marker(h.state))).toBe(false);
}
});
});
describe('implementation handoff recommendation', () => {
test('known harnesses pass their native provider; other hosts get both; nothing switches the session', () => {
expect(generateImplementationModelHandoff(ctxFor('autoplan', 'claude', HOST_PATHS.claude))).toContain('gstack-models" resolve --role implementation --provider anthropic`');
expect(generateImplementationModelHandoff(ctxFor('spec', 'codex'))).toContain('resolve --role implementation --provider openai`');
const other = generateImplementationModelHandoff(ctxFor('plan-eng-review', 'kiro' as Host));
expect(other).toContain('resolve --role implementation`');
expect(other).toContain('one per provider');
for (const text of [other]) {
expect(text).toContain('cannot change this session');
expect(text).not.toMatch(/claude-|gpt-/);
}
});
test('the recommended command resolves offline from current settings without writes', () => {
const h = home('implementation_tier: frontier\n');
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'implementation', '--provider', 'anthropic', '--json'],
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
expect(r.status).toBe(0);
expect(JSON.parse(r.stdout).selections[0]).toMatchObject({ role: 'implementation', requestedModel: 'claude-fable-5-1' });
expect(fs.readdirSync(h.state)).toEqual(['config.yaml']);
});
});
+1 -1
View File
@@ -114,7 +114,7 @@ describe('own-harness review fallback instructions', () => {
test(`${host.name}: ${name} fallback names only the mode its preflight emits`, () => {
const ctx: TemplateContext = { host: host.name, skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', paths: HOST_PATHS[host.name] };
const text = render(ctx);
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
const mode = name === 'plan' || host.name === 'codex' ? 'under_current_harness' : 'under_codex';
const preflight = text.match(/```bash\n([\s\S]*?)\n```/)![1];
expect(preflight).toContain(mode);
expect([...new Set(text.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
+3 -4
View File
@@ -138,8 +138,7 @@ describe('CI workflow clarity regressions', () => {
test('CEO fallback names the current host mode and needs completed findings before the later report exists', () => {
for (const host of ALL_HOST_CONFIGS) {
const source = generateCodexPlanReview({ skillName: 'plan-ceo-review', host: host.name, paths: HOST_PATHS[host.name] } as TemplateContext);
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual(['under_current_harness']);
expect(source).toMatch(/SOURCE=in-host, OUTSIDE_STATUS=unavailable, and STATUS=clean or issues_found/);
expect(source).not.toContain('Sections 1-10/11 and current report');
}
@@ -646,7 +645,7 @@ describe('outside-voice commitment queue', () => {
for (const host of ALL_HOST_CONFIGS) {
const eng = generateCodexPlanReview({ host: host.name, paths: HOST_PATHS[host.name]!, skillName: 'plan-eng-review' } as TemplateContext);
const provider = host.name === 'codex' ? 'Claude Code' : 'Codex';
const mismatch = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
const mismatch = 'under_current_harness';
// B1: the heading also admits `unverified`.
expect(eng).toContain('**If `CODEX_MODE: ready`');
expect(eng).toContain(`— run ${provider}:**`);
@@ -665,7 +664,7 @@ describe('outside-voice commitment queue', () => {
expect(eng).not.toContain('codex exec');
} else {
expect(eng).toContain('codex exec');
expect(eng).toMatch(/then follow \*\*Native fallback\*\*/i);
expect(eng).toContain('_gstack_codex_role_ready exec');
}
}
});
+22 -1
View File
@@ -4,7 +4,7 @@ import * as path from 'node:path';
import * as os from 'node:os';
import { spawnSync } from 'node:child_process';
import {
callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
callerExcerpt, callerProbeReadErrors, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
QA_CALLER_CASES, QA_CALLER_TEST_MS,
qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, unloadedHelperCommand, validateCallerEvidence,
type CallerProbe, type CallerReceipt, type QaCallerFixture,
@@ -87,6 +87,27 @@ function rebuildCheckpoints(observed: ReturnType<typeof evidence>) {
}
describe('caller native-event observer controls', () => {
test('captured instruction Read and probe in one provider response is not consumed instruction evidence', () => {
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-caller-read-probe-mtb1007a.json'), 'utf8'));
const tools = callerTools(captured.events);
const probe = tools.find(tool => tool.name === 'Bash')!;
const resources = ['exploratory', 'system-functional'].map(id => [id, tools.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const);
expect(resources[0][1]!.resultIndex).toBeLessThan(probe.index);
expect(resources[0][1]!.messageId).toBe(probe.messageId);
expect(callerProbeReadErrors(probe, resources)).toEqual([
'probe preceded resource read: exploratory', 'probe preceded resource read: system-functional',
]);
expect(callerProbeReadErrors({ ...probe, messageId: 'separate-response-control' }, resources))
.toEqual(['probe preceded resource read: system-functional']);
const ordered = structuredClone(captured.events);
const command = ordered.splice(2, 2);
command[0].message.id = 'separate-response-control';
ordered.push(...command);
const ready = callerTools(ordered);
expect(callerProbeReadErrors(ready.find(tool => tool.name === 'Bash')!,
['exploratory', 'system-functional'].map(id => [id, ready.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const))).toEqual([]);
});
test.each(['pending-result', 'same-event', 'same-message-id'])('method Read %s cannot authorize a dependent probe', ordering => {
const observed = evidence();
const events = observed.result.transcript as any[];
+11 -1
View File
@@ -59,6 +59,14 @@ function assertPlanExecution(text: string, shared = generateQAExploratory({ host
}
describe('QA probe entry and checkpoint gates', () => {
test('the caller exposes the response boundary before the first QA instruction read', () => {
for (const skillName of ['review', 'ship']) {
const text = generateQAReviewPreflight({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude });
const beforeRead = text.slice(0, text.indexOf('{{QA_RESOURCE:exploratory}}'));
expectMentions(beforeRead, [['reads', 'earlier', 'responses'], ['capture', 'probe'], ['never', 'batch', 'read']], 'caller entry gate');
}
});
test('the native CI preparation excerpt lacks the required acknowledged-read gate', () => {
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-functional-ci-36505065023.json'), 'utf8'));
const output = captured.omittedReadEvents.flatMap(event => event.message.content)
@@ -88,7 +96,9 @@ describe('QA probe entry and checkpoint gates', () => {
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' ');
const body = generateQAReview(ctx).replace(/\s+/g, ' ');
expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i);
expect(preflight).toContain('`sections/...`');
expect(preflight).toContain('`templates/...`');
expectMentions(preflight, [['paths', 'installed qa skill.md', 'not', 'caller', 'product']], 'QA asset resolution');
expect(body).toContain('Read QA\'s `sections/browser-setup.md`');
expect(body).toContain('Read QA\'s `templates/functional-report-template.md`');
if (skillName === 'review') {
+8 -1
View File
@@ -366,7 +366,14 @@ describe('setup --team / --no-team / -q', () => {
fs.writeFileSync(file, content, { mode: 0o755 });
};
try {
for (const rel of ['setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names', 'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh', 'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts', 'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts']) {
for (const rel of [
'setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names',
'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh',
'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts',
'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts',
'lib/claude-code-migration.ts', 'lib/model-policy.ts', 'lib/model-catalog.ts',
'lib/model-policy-notice.ts', 'lib/state-root.ts',
]) {
const dest = path.join(cwd, rel);
fs.mkdirSync(path.dirname(dest), { recursive: true });
fs.copyFileSync(path.join(ROOT, rel), dest);