mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
feat: add configurable model tiers and advisory freshness
This commit is contained in:
1 parent
db745675bd
commit
5ffc050659
78 files changed
+5106
-325
No files matched your search
@@ -0,0 +1,49 @@
|
||||
name: Model-policy freshness
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 11 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: model-policy-freshness
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
advisory:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 8
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile
|
||||
- name: Check and publish main advisory
|
||||
if: github.ref == 'refs/heads/main'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json
|
||||
run: bun scripts/model-policy-freshness.ts --publish
|
||||
- name: Check non-main without publication
|
||||
if: github.ref != 'refs/heads/main'
|
||||
env:
|
||||
MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json
|
||||
run: bun scripts/model-policy-freshness.ts
|
||||
- name: Preserve evidence report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
|
||||
with:
|
||||
name: model-policy-freshness-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/model-policy-freshness.json
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
+3
-2
@@ -173,8 +173,9 @@ defaults are `claude-fable-5-1`, resolved through `lib/eval-model.ts`:
|
||||
|
||||
Warmup stays on `claude-haiku-4-5`; distill stays on
|
||||
`claude-haiku-4-5-20251001`. Explicit test and historical benchmark model
|
||||
selections still win. Known frontier defaults are maintained in releases;
|
||||
there is no automatic model discovery. Paid-run costs shown below are
|
||||
selections still win. These eval defaults stay pinned independently from the
|
||||
[runtime model tiers](docs/model-policy.md); the tier freshness workflow does
|
||||
not discover or change evaluation models automatically. Paid-run costs shown below are
|
||||
historical estimates from before this default change, not measurements of
|
||||
the new defaults.
|
||||
|
||||
|
||||
@@ -165,8 +165,18 @@ override applies to that run only; set `model` in your Codex `config.toml` to
|
||||
make it stick across upgrades. After changing your Codex model, rerun
|
||||
`./setup --host codex` to regenerate the skills.
|
||||
|
||||
**Which Codex model gstack uses.** For every Codex call (outside voices,
|
||||
`/codex`, review and ship adversarial passes), gstack picks the model in this
|
||||
**Plan-review and implementation tiers.** Plan workflows use an independent
|
||||
`frontier` reviewer: Fable 5.1 from Codex, or GPT-6 Astra from Claude Code.
|
||||
Implementation handoffs recommend the `smart` tier: Opus 5.5 or GPT-6.1 Sol.
|
||||
Choose a different tier, pin either provider's model, or use `host` mode to
|
||||
preserve native model settings. The read-only `gstack-models` command explains
|
||||
the effective choices; changing these settings does not require regenerating
|
||||
skills or switch your running session. A weekly maintenance check flags changed
|
||||
official recommendations and retirement notices without automatically changing
|
||||
models. See [model policy: setup, overrides and freshness](docs/model-policy.md).
|
||||
|
||||
**Other Codex calls keep their existing selection.** Without the explicit
|
||||
`plan-review` role (including ordinary review and ship adversarial passes), gstack picks the model in this
|
||||
order: a model you name for that request, then `GSTACK_CODEX_MODEL`, then
|
||||
`model` in your Codex `config.toml` (for native `codex review`, `review_model`
|
||||
first; a custom `CODEX_HOME` is honored), and only then gstack's default,
|
||||
@@ -177,7 +187,7 @@ repair message and reports the outside review as unavailable. It never silently
|
||||
switches to its default. Nested Codex reviews also run with installed skills
|
||||
hidden (`-c skills.include_instructions=false`), so a review cannot turn into a
|
||||
whole nested skill run. Runtime model selection is separate from the setup-time
|
||||
behavioral profile above. `/claude-code` (`gstack-claude-code`
|
||||
behavioral profile above. Without the plan-review role, `/claude-code` (`gstack-claude-code`
|
||||
on Codex) preserves Claude's configured model. Set `GSTACK_CLAUDE_MODEL=<model>`
|
||||
or name a model in your request to override it for the invocation, including
|
||||
resumed consultations. See [eval defaults and overrides](CONTRIBUTING.md#testing--evals)
|
||||
|
||||
+18
-45
@@ -754,56 +754,27 @@ Review skills will load at each phase entry. Starting full review pipeline with
|
||||
|
||||
```bash
|
||||
|
||||
# Codex preflight: one block (functions sourced here don't persist to later blocks).
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
|
||||
_gstack_helper_error=""
|
||||
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
|
||||
if [ "$_CODEX_CFG" = "disabled" ]; then
|
||||
_CODEX_MODE="disabled"
|
||||
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
_CODEX_MODE="under_codex"
|
||||
elif ! command -v codex >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
|
||||
elif [ -n "$_gstack_helper_error" ]; then
|
||||
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
|
||||
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
|
||||
else
|
||||
# The free sandbox check runs before the paid model probe. Probe code 2 means
|
||||
# the CLI cannot execute at all, a different fix from an unusable model.
|
||||
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
|
||||
if [ "$_CODEX_MP" -eq 3 ]; then
|
||||
_CODEX_MODE="sandbox_unavailable"
|
||||
elif [ "$_CODEX_MP" -eq 2 ]; then
|
||||
_CODEX_MODE="broken_install"
|
||||
elif [ "$_CODEX_MP" -eq 4 ]; then
|
||||
_CODEX_MODE="quota_exhausted"
|
||||
elif [ "$_CODEX_MP" -ne 0 ]; then
|
||||
_CODEX_MODE="model_unusable"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
|
||||
_CODEX_MODE="unverified"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
|
||||
_CODEX_MODE="unverified (rate_limited)"
|
||||
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
|
||||
if [ "$_OUTSIDE_CFG" = disabled ]; then
|
||||
echo 'CODEX_MODE: disabled'
|
||||
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
|
||||
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
|
||||
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
|
||||
else
|
||||
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
|
||||
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
|
||||
fi
|
||||
exit 78
|
||||
fi
|
||||
); then
|
||||
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
|
||||
else
|
||||
echo 'CODEX_MODE: under_current_harness'
|
||||
fi
|
||||
echo "CODEX_MODE: $_CODEX_MODE"
|
||||
```
|
||||
|
||||
Branch on the echoed `CODEX_MODE`:
|
||||
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the Codex passes only; the Claude adversarial subagent below STILL runs (it is free and fast). Print: "Codex passes skipped (codex_reviews disabled) — running Claude adversarial only."
|
||||
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider.
|
||||
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Keep the required Claude adversarial pass; do not dispatch a duplicate. The ~10s round trip is cached for 1h.
|
||||
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Keep the required Claude adversarial pass; do not dispatch a duplicate.
|
||||
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled skips only the outside CLI; retain the native pass. Non-ready means missing outside coverage. Keep the required native pass without duplicating it. Never substitute another external provider.
|
||||
|
||||
Disabled/unavailable retains applicable native passes. Recheck each outside dispatch.
|
||||
Record provider and completed/unavailable/disabled/skipped per phase; CEO covers
|
||||
@@ -990,4 +961,6 @@ Retain the historical review-log skill ID; add `"host":"claude","outside_provide
|
||||
Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status,
|
||||
native completion, findings, and partial coverage. Replace N with actual counts.
|
||||
|
||||
**Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
|
||||
|
||||
Suggest next step: `/ship` when ready to create the PR.
|
||||
@@ -464,4 +464,6 @@ disabled or skipped. Never carry success across phases/runs; preserve modelUsage
|
||||
Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status,
|
||||
native completion, findings, and partial coverage. Replace N with actual counts.
|
||||
|
||||
{{IMPLEMENTATION_MODEL_HANDOFF}}
|
||||
|
||||
Suggest next step: `/ship` when ready to create the PR.
|
||||
@@ -111,8 +111,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -70,8 +70,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -74,8 +74,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -63,8 +63,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
+234
-36
@@ -10,6 +10,8 @@
|
||||
# _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off
|
||||
# _gstack_codex_model_probe — round-trip probe of the selected model (#2477);
|
||||
# exit 4 = the account's Codex usage limit
|
||||
# _gstack_codex_lock_acquire — bounded mkdir lock with abandoned-owner recovery
|
||||
# _gstack_codex_role_ready — plan-review selection, auth, sandbox and model probe in one shell
|
||||
# _gstack_codex_version_check — warn on known-bad Codex CLI versions
|
||||
# _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog
|
||||
# _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/
|
||||
@@ -114,33 +116,65 @@ _gstack_codex_select_model() {
|
||||
# One selection record per invocation kind, shared by the probe and the
|
||||
# dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL,
|
||||
# Codex config.toml (review_model first for native review), gstack default.
|
||||
# With $3 = plan-review the plan-review policy ranks the choice instead and
|
||||
# its record carries role, tier, source and source-aware repair; $4 names the
|
||||
# target repository whose project settings can route to a custom endpoint.
|
||||
# The TOML is read by the TypeScript resolver; bash never parses it.
|
||||
local _kind="${1:-exec}" _explicit="${2:-}" _line _rc
|
||||
local _kind="${1:-exec}" _explicit="${2:-}" _role="${3:-}" _cwd="${4:-}" _line _rc _rest
|
||||
case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac
|
||||
case "$_role" in ''|plan-review) ;; *) echo "ERROR: _gstack_codex_select_model role must be plan-review" >&2; return 2 ;; esac
|
||||
if ! command -v bun >/dev/null 2>&1; then
|
||||
echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2
|
||||
return 1
|
||||
fi
|
||||
if [ -n "$_explicit" ]; then
|
||||
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit")
|
||||
else
|
||||
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind")
|
||||
fi
|
||||
# Function-local argv: each value stays one literal argument in bash and zsh.
|
||||
set -- --runtime "$_kind"
|
||||
[ -n "$_role" ] && set -- "$@" --role "$_role"
|
||||
[ -n "$_explicit" ] && set -- "$@" --explicit "$_explicit"
|
||||
[ -n "$_cwd" ] && set -- "$@" --cwd "$_cwd"
|
||||
_line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" "$@")
|
||||
_rc=$?
|
||||
_GSTACK_CODEX_SEL="${_line%%$'\t'*}"
|
||||
_GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}"
|
||||
_rest="${_line#*$'\t'}"
|
||||
_GSTACK_CODEX_SEL_TIER=""
|
||||
_GSTACK_CODEX_SEL_REPAIR=""
|
||||
if [ -n "$_role" ]; then
|
||||
_GSTACK_CODEX_SEL_SRC="${_rest%%$'\t'*}"
|
||||
_rest="${_rest#*$'\t'}"
|
||||
_GSTACK_CODEX_SEL_TIER="${_rest%%$'\t'*}"
|
||||
_GSTACK_CODEX_SEL_REPAIR="${_rest#*$'\t'}"
|
||||
else
|
||||
_GSTACK_CODEX_SEL_SRC="$_rest"
|
||||
fi
|
||||
case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac
|
||||
if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then
|
||||
_GSTACK_CODEX_SEL=""
|
||||
_GSTACK_CODEX_SEL_SRC=""
|
||||
_GSTACK_CODEX_SEL_ROLE=""
|
||||
echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2
|
||||
return 1
|
||||
fi
|
||||
_GSTACK_CODEX_SEL_KIND="$_kind"
|
||||
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
||||
_GSTACK_CODEX_SEL_ROLE="$_role"
|
||||
_GSTACK_CODEX_SEL_CWD="$_cwd"
|
||||
if [ -n "$_role" ]; then
|
||||
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; role: $_role, tier: $_GSTACK_CODEX_SEL_TIER; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
||||
else
|
||||
echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2
|
||||
fi
|
||||
_gstack_codex_sandbox_mode
|
||||
}
|
||||
|
||||
_gstack_codex_role_ready() {
|
||||
# $1 exec|review, $2 target repository. Resolves the plan-review model once and
|
||||
# checks auth, the sandbox and that model; the caller dispatches the same record.
|
||||
_gstack_codex_select_model "$1" '' plan-review "${2:-}" || return 1
|
||||
_gstack_codex_auth_probe || return 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || return 1
|
||||
_gstack_codex_version_check
|
||||
_gstack_codex_model_probe "$1"
|
||||
}
|
||||
|
||||
# --- Sandbox (B1) -----------------------------------------------------------
|
||||
|
||||
_gstack_codex_sandbox_mode() {
|
||||
@@ -246,7 +280,7 @@ _gstack_codex_model_probe() {
|
||||
# auth just measures the auth failure again.
|
||||
local _kind="${1:-exec}"
|
||||
if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then
|
||||
if ! _gstack_codex_select_model "$_kind"; then
|
||||
if ! _gstack_codex_select_model "$_kind" "" "${_GSTACK_CODEX_SEL_ROLE:-}" "${_GSTACK_CODEX_SEL_CWD:-}"; then
|
||||
echo "MODEL_UNUSABLE"
|
||||
echo "HINT: the requested Codex model is not a valid model id; fix it as described above."
|
||||
return 1
|
||||
@@ -286,36 +320,42 @@ _gstack_codex_model_probe() {
|
||||
local _now
|
||||
_now=$(date +%s 2>/dev/null || echo 0)
|
||||
local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md"
|
||||
if [ -n "$_cache" ] && [ -f "$_cache" ]; then
|
||||
local _c_line _c_status _c_ts _c_sig
|
||||
_c_line=$(head -1 "$_cache" 2>/dev/null)
|
||||
_c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1)
|
||||
_c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2)
|
||||
_c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3)
|
||||
case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac
|
||||
if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
|
||||
echo "MODEL_OK (cached)"
|
||||
return 0
|
||||
# Up to 16 signatures share one cache file. Same-signature misses serialize
|
||||
# on a bounded lock and recheck the cache, so concurrent callers pay for one
|
||||
# round trip. A lock timeout or unreadable cache is unverified readiness,
|
||||
# never a duplicate paid probe; the review's own result still decides.
|
||||
local _lock="" _rc
|
||||
if [ -n "$_cache" ]; then
|
||||
_gstack_codex_probe_cache_hit; _rc=$?
|
||||
[ "$_rc" -lt 10 ] && return "$_rc"
|
||||
if [ "$_rc" -eq 11 ]; then _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; fi
|
||||
_lock="$_gstack_home/.codex-model-probe.locks/$_sig"
|
||||
if ! mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null; then
|
||||
_gstack_codex_probe_unverified "cannot create the probe lock under $_gstack_home"; return 0
|
||||
fi
|
||||
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry"
|
||||
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
||||
echo "MODEL_QUOTA_EXHAUSTED (cached)"
|
||||
sed -n '2,4p' "$_cache" 2>/dev/null
|
||||
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
|
||||
return 4
|
||||
fi
|
||||
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
||||
echo "MODEL_UNUSABLE (cached)"
|
||||
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
||||
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
||||
return 1
|
||||
_gstack_codex_lock_acquire "$_lock" "${_GSTACK_CODEX_LOCK_WAIT:-45}" "${_GSTACK_CODEX_LOCK_STALE:-90}"; _rc=$?
|
||||
if [ "$_rc" -eq 1 ]; then _gstack_codex_probe_unverified "another probe of this model still holds its lock"; return 0; fi
|
||||
if [ "$_rc" -ne 0 ]; then _gstack_codex_probe_unverified "cannot record ownership of the probe lock under $_gstack_home"; return 0; fi
|
||||
_gstack_codex_probe_cache_hit; _rc=$?
|
||||
if [ "$_rc" -lt 10 ] || [ "$_rc" -eq 11 ]; then
|
||||
_gstack_codex_lock_release "$_lock"
|
||||
[ "$_rc" -eq 11 ] && { _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; }
|
||||
return "$_rc"
|
||||
fi
|
||||
fi
|
||||
_gstack_codex_probe_round_trip; _rc=$?
|
||||
[ -n "$_lock" ] && _gstack_codex_lock_release "$_lock"
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
_gstack_codex_probe_round_trip() {
|
||||
# The paid round trip and its classification. Called only by
|
||||
# _gstack_codex_model_probe; reads its locals (_model, _sig, _cache, ...).
|
||||
local _out _code
|
||||
_out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" </dev/null 2>&1)
|
||||
_code=$?
|
||||
if [ "$_code" -eq 0 ]; then
|
||||
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
|
||||
_gstack_codex_probe_cache_store MODEL_OK
|
||||
echo "MODEL_OK"
|
||||
return 0
|
||||
fi
|
||||
@@ -335,7 +375,7 @@ _gstack_codex_model_probe() {
|
||||
fi
|
||||
_q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3)
|
||||
if [ -n "$_q_lines" ]; then
|
||||
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_QUOTA_EXHAUSTED %s %s\n%s\n' "$_now" "$_sig" "$_q_lines" > "$_cache" 2>/dev/null; }
|
||||
_gstack_codex_probe_cache_store MODEL_QUOTA_EXHAUSTED "$_q_lines"
|
||||
echo "MODEL_QUOTA_EXHAUSTED"
|
||||
printf '%s\n' "$_q_lines"
|
||||
echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted"
|
||||
@@ -357,7 +397,7 @@ _gstack_codex_model_probe() {
|
||||
# base_url answers a bare 404; neither is network luck, so neither may fall
|
||||
# through to the inconclusive branch below and report ready (#2843).
|
||||
if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then
|
||||
[ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; }
|
||||
_gstack_codex_probe_cache_store MODEL_UNUSABLE
|
||||
echo "MODEL_UNUSABLE"
|
||||
printf '%s\n' "$_out" | grep -iE "model|404" | head -3
|
||||
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
||||
@@ -366,10 +406,14 @@ _gstack_codex_model_probe() {
|
||||
_gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true
|
||||
elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then
|
||||
echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong."
|
||||
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
|
||||
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
|
||||
echo "HINT: choose a current model ($_GSTACK_CODEX_SEL_REPAIR), or fix that provider's base_url."
|
||||
else
|
||||
echo "HINT: choose a current model (GSTACK_CODEX_MODEL=<supported-model> or model in config.toml), or fix that provider's base_url."
|
||||
fi
|
||||
_gstack_codex_log_event "codex_model_retired" 2>/dev/null || true
|
||||
else
|
||||
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in $_codex_home/config.toml, or name one for this request."
|
||||
_gstack_codex_model_repair_hint
|
||||
_gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true
|
||||
fi
|
||||
return 1
|
||||
@@ -400,6 +444,160 @@ _gstack_codex_model_probe() {
|
||||
return 0
|
||||
}
|
||||
|
||||
_gstack_codex_model_repair_hint() {
|
||||
# Repair for the record actually selected: a plan-review record names its
|
||||
# winning source (request, env, tier override, catalog or host mode).
|
||||
if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then
|
||||
echo "HINT: choose a model your account can use: $_GSTACK_CODEX_SEL_REPAIR"
|
||||
else
|
||||
echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=<supported-model>, set model in ${_codex_home:-\$CODEX_HOME}/config.toml, or name one for this request."
|
||||
fi
|
||||
}
|
||||
|
||||
_gstack_codex_probe_unverified() {
|
||||
_GSTACK_CODEX_PROBE_STATE="inconclusive"
|
||||
echo "MODEL_PROBE_INCONCLUSIVE ($1) — CODEX_MODE: unverified; no duplicate probe was made, and the review's own result is still checked."
|
||||
}
|
||||
|
||||
_gstack_codex_probe_cache_hit() {
|
||||
# Cache file: one "STATUS EPOCH SIGNATURE" line per entry, newest last, each
|
||||
# followed by its tab-prefixed Codex lines. The pre-v2 one-entry file has the
|
||||
# same first line, so it migrates by TTL; its unprefixed lines are ignored.
|
||||
# Returns 0/1/4 after printing a hit, 10 on a miss, 11 when unreadable.
|
||||
[ -e "$_cache" ] || return 10
|
||||
{ [ -f "$_cache" ] && [ -r "$_cache" ]; } || return 11
|
||||
local _hit _head _c_status _c_ts _detail=""
|
||||
_hit=$(awk -v sig="$_sig" '
|
||||
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ { cur = ($3 == sig); if (cur) out = $1 " " $2; next }
|
||||
/^\t/ { if (cur) out = out "\n" substr($0, 2); next }
|
||||
{ cur = 0 }
|
||||
END { if (out != "") print out }' "$_cache" 2>/dev/null) || return 11
|
||||
[ -n "$_hit" ] || return 10
|
||||
_head="${_hit%%$'\n'*}"
|
||||
[ "$_head" != "$_hit" ] && _detail="${_hit#*$'\n'}"
|
||||
_c_status="${_head%% *}"
|
||||
_c_ts="${_head#* }"
|
||||
case "$_c_ts" in ''|*[!0-9]*) return 10 ;; esac
|
||||
[ "$_c_ts" -le "$_now" ] || return 10
|
||||
if [ "$_c_status" = "MODEL_OK" ] && [ $((_now - _c_ts)) -lt 3600 ]; then
|
||||
echo "MODEL_OK (cached)"
|
||||
return 0
|
||||
fi
|
||||
[ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && return 10
|
||||
if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
||||
echo "MODEL_QUOTA_EXHAUSTED (cached)"
|
||||
[ -n "$_detail" ] && printf '%s\n' "$_detail"
|
||||
echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted"
|
||||
return 4
|
||||
fi
|
||||
if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ $((_now - _c_ts)) -lt 900 ]; then
|
||||
echo "MODEL_UNUSABLE (cached)"
|
||||
echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)."
|
||||
_gstack_codex_model_repair_hint
|
||||
return 1
|
||||
fi
|
||||
return 10
|
||||
}
|
||||
|
||||
_gstack_codex_probe_cache_store() {
|
||||
# $1 status, $2 optional Codex lines. Rewrites the file under a short write
|
||||
# lock and publishes it with one rename: this signature's entry replaces any
|
||||
# older one, expired entries drop, and at most 16 entries remain. A failure
|
||||
# leaves the previous file intact; the probe's own result still stands.
|
||||
[ -n "$_cache" ] || return 0
|
||||
[ -d "$_cache" ] && return 1
|
||||
local _w="$_gstack_home/.codex-model-probe.locks/write" _tmp="$_cache.tmp.$$" _wrc=0
|
||||
mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null || return 1
|
||||
_gstack_codex_lock_acquire "$_w" 5 30 || return 1
|
||||
: > "$_tmp" 2>/dev/null || _wrc=1
|
||||
if [ "$_wrc" -eq 0 ] && [ -f "$_cache" ]; then
|
||||
awk -v now="$_now" -v sig="$_sig" -v keep=15 '
|
||||
/^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ {
|
||||
ttl = ($1 == "MODEL_OK") ? 3600 : 900
|
||||
cur = ($3 != sig && $2 <= now && now - $2 < ttl)
|
||||
if (cur) entry[++n] = $0
|
||||
next
|
||||
}
|
||||
/^\t/ { if (cur) entry[n] = entry[n] "\n" $0; next }
|
||||
{ cur = 0 }
|
||||
END { for (i = (n > keep ? n - keep + 1 : 1); i <= n; i++) print entry[i] }' "$_cache" > "$_tmp" 2>/dev/null || _wrc=1
|
||||
fi
|
||||
if [ "$_wrc" -eq 0 ]; then
|
||||
{
|
||||
printf '%s %s %s\n' "$1" "$_now" "$_sig"
|
||||
[ -z "${2:-}" ] || printf '%s\n' "$2" | awk '{ print "\t" $0 }'
|
||||
} >> "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_cache" 2>/dev/null || _wrc=1
|
||||
fi
|
||||
rm -f "$_tmp" 2>/dev/null
|
||||
_gstack_codex_lock_release "$_w"
|
||||
return "$_wrc"
|
||||
}
|
||||
|
||||
# --- Bounded lock -----------------------------------------------------------
|
||||
|
||||
_gstack_codex_lock_acquire() {
|
||||
# $1 lock directory, $2 seconds to wait, $3 seconds after which even a live
|
||||
# owner is abandoned. mkdir is atomic on every platform bash runs on. A dead
|
||||
# owner on this host is recovered at once; a missing owner file is the
|
||||
# creation window and becomes stale after 5 seconds. 0 = held, 1 = timed
|
||||
# out, 2 = the lock or its owner record cannot be created.
|
||||
local _dir="$1" _tries=$(( ${2:-45} * 10 )) _stale="${3:-90}" _host _owner _pid _ohost _ts _now _i=0 _dead
|
||||
_host=$(uname -n 2>/dev/null || echo unknown)
|
||||
while :; do
|
||||
if mkdir "$_dir" 2>/dev/null; then
|
||||
printf '%s %s %s\n' "$$" "$_host" "$(date +%s 2>/dev/null || echo 0)" > "$_dir/owner" 2>/dev/null && return 0
|
||||
rm -f "$_dir/owner" 2>/dev/null
|
||||
rmdir "$_dir" 2>/dev/null
|
||||
return 2
|
||||
fi
|
||||
[ -d "$_dir" ] || return 2
|
||||
_now=$(date +%s 2>/dev/null || echo 0)
|
||||
_owner=$(cat "$_dir/owner" 2>/dev/null)
|
||||
_dead=0
|
||||
if [ -n "$_owner" ]; then
|
||||
read -r _pid _ohost _ts <<< "$_owner"
|
||||
case "$_pid" in ''|*[!0-9]*) _pid=0 ;; esac
|
||||
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
|
||||
if [ "$_ohost" = "$_host" ] && ! kill -0 "$_pid" 2>/dev/null; then _dead=1; fi
|
||||
[ $((_now - _ts)) -ge "$_stale" ] && _dead=1
|
||||
else
|
||||
_ts=$(stat -c %Y "$_dir" 2>/dev/null || stat -f %m "$_dir" 2>/dev/null || echo 0)
|
||||
case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac
|
||||
[ $((_now - _ts)) -ge 5 ] && _dead=1
|
||||
fi
|
||||
[ "$_dead" -eq 1 ] && _gstack_codex_lock_break "$_dir" "$_owner"
|
||||
[ "$_i" -ge "$_tries" ] && return 1
|
||||
_i=$((_i + 1))
|
||||
sleep 0.1
|
||||
done
|
||||
}
|
||||
|
||||
_gstack_codex_lock_break() {
|
||||
# Breakers serialize on $1.break and remove the lock only if it still has the
|
||||
# owner they judged abandoned, so a lock re-taken meanwhile survives.
|
||||
local _dir="$1" _seen="$2" _bt _now
|
||||
if mkdir "$_dir.break" 2>/dev/null; then
|
||||
[ "$(cat "$_dir/owner" 2>/dev/null)" = "$_seen" ] && rm -rf "$_dir" 2>/dev/null
|
||||
rmdir "$_dir.break" 2>/dev/null
|
||||
return 0
|
||||
fi
|
||||
_now=$(date +%s 2>/dev/null || echo 0)
|
||||
_bt=$(stat -c %Y "$_dir.break" 2>/dev/null || stat -f %m "$_dir.break" 2>/dev/null || echo "$_now")
|
||||
case "$_bt" in ''|*[!0-9]*) _bt="$_now" ;; esac
|
||||
[ $((_now - _bt)) -ge 10 ] && rmdir "$_dir.break" 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
|
||||
_gstack_codex_lock_release() {
|
||||
# Remove only a lock this shell still owns.
|
||||
local _host
|
||||
_host=$(uname -n 2>/dev/null || echo unknown)
|
||||
case "$(cat "$1/owner" 2>/dev/null)" in
|
||||
"$$ $_host "*) rm -rf "$1" 2>/dev/null ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- Version check ----------------------------------------------------------
|
||||
|
||||
_gstack_codex_version_check() {
|
||||
|
||||
+44
-2
@@ -151,6 +151,20 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne
|
||||
# # ID and re-renders; ./setup --claude-model claude
|
||||
# # returns to the generic overlay.
|
||||
#
|
||||
# ─── Model policy (gstack-models shows the effective choices) ───────
|
||||
# plan_review_tier: frontier # frontier | smart | host — model tier for the
|
||||
# # opposing-provider plan review. host = the
|
||||
# # Codex / Claude Code CLI settings choose.
|
||||
# # GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL win.
|
||||
# implementation_tier: smart # frontier | smart — tier gstack recommends for
|
||||
# # implementing an approved plan (never switches
|
||||
# # your session).
|
||||
# model_frontier_claude: # per-tier model IDs; empty = the shipped catalog
|
||||
# model_frontier_openai: # recommendation (gstack-models list). Invalid
|
||||
# model_smart_claude: # values are REJECTED and the stored value kept;
|
||||
# model_smart_openai: # unset restores the shipped recommendation.
|
||||
# # Docs: docs/model-policy.md
|
||||
#
|
||||
# ─── Advanced ────────────────────────────────────────────────────────
|
||||
# codex_reviews: enabled # Workflow outside review (Codex or Claude Code by harness).
|
||||
# # enabled: /review, /ship, /document-release, plan reviews,
|
||||
@@ -278,6 +292,9 @@ lookup_default() {
|
||||
ship_measure_sweep_cases) echo "5" ;;
|
||||
ship_measure_sweep_budget_usd) echo "150" ;;
|
||||
ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in)
|
||||
plan_review_tier) echo "frontier" ;; # frontier | smart | host — lib/model-policy.ts reads it strictly
|
||||
implementation_tier) echo "smart" ;; # frontier | smart
|
||||
model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai) echo "" ;; # empty = shipped catalog ID (gstack-models list)
|
||||
# Unknown key: exit non-zero instead of printing "". The fallback pattern
|
||||
# the preambles use,
|
||||
# VAR=$(gstack-config get <key> 2>/dev/null || echo "<default>")
|
||||
@@ -678,6 +695,27 @@ case "${1:-}" in
|
||||
echo "Error: codex_reviews '$VALUE' not recognized. Valid values: enabled, disabled. Existing value left unchanged." >&2
|
||||
exit 1
|
||||
fi
|
||||
case "$KEY" in
|
||||
plan_review_tier|implementation_tier|model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai)
|
||||
_mp_doc="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#model-policy-config"
|
||||
VALUE="${VALUE#"${VALUE%%[![:space:]]*}"}"
|
||||
VALUE="${VALUE%"${VALUE##*[![:space:]]}"}"
|
||||
_mp_valid=1
|
||||
case "$KEY" in
|
||||
plan_review_tier) case "$VALUE" in frontier|smart|host) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart, host." ;; esac ;;
|
||||
implementation_tier) case "$VALUE" in frontier|smart) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart." ;; esac ;;
|
||||
*_claude) printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[][A-Za-z0-9._:/@+-]{1,512}$' || { _mp_valid=""; _mp_rule="An Anthropic model ID is 1-512 characters from A-Z a-z 0-9 . _ : / @ + [ ] -."; } ;;
|
||||
*_openai) printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[A-Za-z0-9._:/-]{1,100}$' || { _mp_valid=""; _mp_rule="An OpenAI model ID is 1-100 characters from A-Z a-z 0-9 . _ : / -."; } ;;
|
||||
esac
|
||||
if [ -z "$VALUE" ]; then
|
||||
echo "Error: $KEY value is empty. Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$_mp_valid" ]; then
|
||||
echo "Error: $KEY '$VALUE' not recognized. $_mp_rule Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2
|
||||
exit 1
|
||||
fi ;;
|
||||
esac
|
||||
# The measure-loop controls bound paid spend: reject a typo instead of
|
||||
# coercing it, so a bad value never changes how much /ship may spend.
|
||||
case "$KEY" in
|
||||
@@ -882,7 +920,9 @@ case "${1:-}" in
|
||||
timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \
|
||||
browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \
|
||||
ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \
|
||||
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do
|
||||
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \
|
||||
plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \
|
||||
model_smart_claude model_smart_openai; do
|
||||
gstack_config_select "$KEY"
|
||||
VALUE="$_gstack_cfg_value"
|
||||
SOURCE="default"
|
||||
@@ -905,7 +945,9 @@ case "${1:-}" in
|
||||
timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \
|
||||
browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \
|
||||
ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \
|
||||
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do
|
||||
ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \
|
||||
plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \
|
||||
model_smart_claude model_smart_openai; do
|
||||
printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")"
|
||||
done
|
||||
;;
|
||||
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bun
|
||||
import { modelsMain } from '../lib/model-policy-cli.ts';
|
||||
|
||||
const result = modelsMain(process.argv.slice(2));
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
process.exit(result.code);
|
||||
@@ -53,6 +53,9 @@ Set `GSTACK_CLAUDE_MODEL=<model>` for an explicit override, including resumed
|
||||
consultations. If the user names a model, pass that value through this environment
|
||||
variable for every runner call. Without an override, retain Claude's configured
|
||||
model; harness routing never chooses a model family.
|
||||
Only for an explicit `--role plan-review` (e.g. `/claude-code challenge --role plan-review`),
|
||||
append `--role plan-review` to every runner call; the runner prints `CLAUDE_MODEL:` and
|
||||
passes that one policy model (none in host mode).
|
||||
|
||||
Do not infer authentication state from credential files or environment variables.
|
||||
Run the actual runner invocation in the host's normal execution context. On a
|
||||
|
||||
+5
-1
@@ -473,9 +473,13 @@ then `gpt-6-astra`. Each call prints `CODEX_MODEL: <model> (<kind>; source: ...)
|
||||
For a named model, pass it as the second argument of every `_gstack_codex_select_model`
|
||||
call, and run `_gstack_codex_select_model exec '<model>'` before the probe below. An
|
||||
invalid or unavailable choice stops with a repair message, never the default.
|
||||
Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`),
|
||||
set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then
|
||||
probes and dispatches the policy model.
|
||||
|
||||
```bash
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_ROLE=''
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || { echo "HELPER_UNAVAILABLE"; exit 1; }
|
||||
|
||||
# GSTACK_ACTIVE_HOST names the harness, never the model.
|
||||
@@ -491,7 +495,7 @@ fi
|
||||
if ! _gstack_codex_auth_probe >/dev/null; then
|
||||
_gstack_codex_log_event "codex_auth_failed"
|
||||
echo "AUTH_FAILED"
|
||||
elif _gstack_codex_sandbox_preflight; then # free; Linux only
|
||||
elif _gstack_codex_sandbox_preflight && [ -z "$_CODEX_ROLE" ]; then # free; Linux only
|
||||
_gstack_codex_model_probe # ~10s round trip on first run, cached 1h
|
||||
fi
|
||||
_gstack_codex_version_check # warns if known-bad, non-blocking
|
||||
|
||||
+5
-1
@@ -74,16 +74,20 @@ then `gpt-6-astra`. Each call prints `CODEX_MODEL: <model> (<kind>; source: ...)
|
||||
For a named model, pass it as the second argument of every `_gstack_codex_select_model`
|
||||
call, and run `_gstack_codex_select_model exec '<model>'` before the probe below. An
|
||||
invalid or unavailable choice stops with a repair message, never the default.
|
||||
Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`),
|
||||
set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then
|
||||
probes and dispatches the policy model.
|
||||
|
||||
```bash
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_ROLE=''
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || { echo "HELPER_UNAVAILABLE"; exit 1; }
|
||||
|
||||
{{OUTSIDE_SELF_GUARD:codex}}
|
||||
if ! _gstack_codex_auth_probe >/dev/null; then
|
||||
_gstack_codex_log_event "codex_auth_failed"
|
||||
echo "AUTH_FAILED"
|
||||
elif _gstack_codex_sandbox_preflight; then # free; Linux only
|
||||
elif _gstack_codex_sandbox_preflight && [ -z "$_CODEX_ROLE" ]; then # free; Linux only
|
||||
_gstack_codex_model_probe # ~10s round trip on first run, cached 1h
|
||||
fi
|
||||
_gstack_codex_version_check # warns if known-bad, non-blocking
|
||||
|
||||
@@ -49,7 +49,9 @@ fi
|
||||
[ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
|
||||
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
|
||||
@@ -40,7 +40,9 @@ fi
|
||||
[ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/<prompt-file-name>"
|
||||
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
|
||||
_gstack_codex_timeout_wrapper 540 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c "
|
||||
|
||||
@@ -102,7 +102,9 @@ cd "$_REPO_ROOT" || exit 1
|
||||
PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/<prompt-file-name>"
|
||||
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
_SID=""
|
||||
if [ "$_CODEX_MODE" = "resume" ]; then
|
||||
_SID=$(cat .context/codex-session-id 2>/dev/null)
|
||||
|
||||
@@ -93,7 +93,9 @@ cd "$_REPO_ROOT" || exit 1
|
||||
PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/<prompt-file-name>"
|
||||
[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; }
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
_SID=""
|
||||
if [ "$_CODEX_MODE" = "resume" ]; then
|
||||
_SID=$(cat .context/codex-session-id 2>/dev/null)
|
||||
|
||||
@@ -48,7 +48,9 @@ _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo"
|
||||
cd "$_REPO_ROOT"
|
||||
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model review || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model review '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe review || exit $?
|
||||
# The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST
|
||||
# and a stall surfaces as a diagnosable exit 124 with an explicit message,
|
||||
# never as a silent harness kill that downstream reads as "no findings".
|
||||
@@ -104,7 +106,9 @@ FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/<focus-file-name>"
|
||||
[ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; }
|
||||
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
{
|
||||
printf '%s\n' "IMPORTANT: Do NOT read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. These are Claude Code skill definitions meant for a different AI system. Do not invoke any installed skill (Codex home skills/, .agents/); answer directly. Do NOT modify agents/openai.yaml. Stay focused on repository code only."
|
||||
|
||||
@@ -46,7 +46,9 @@ _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo"
|
||||
cd "$_REPO_ROOT"
|
||||
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model review || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model review '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe review || exit $?
|
||||
# The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST
|
||||
# and a stall surfaces as a diagnosable exit 124 with an explicit message,
|
||||
# never as a silent harness kill that downstream reads as "no findings".
|
||||
@@ -95,7 +97,9 @@ FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/<focus-file-name>"
|
||||
[ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; }
|
||||
~/.claude/skills/gstack/bin/gstack-review-log --start codex-review
|
||||
source ~/.claude/skills/gstack/bin/gstack-codex-probe || exit 1
|
||||
_gstack_codex_select_model exec || exit 1
|
||||
_CODEX_ROLE=''
|
||||
_gstack_codex_select_model exec '' "$_CODEX_ROLE" "$_REPO_ROOT" || exit 1
|
||||
[ -z "$_CODEX_ROLE" ] || _gstack_codex_model_probe exec || exit $?
|
||||
_PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; }
|
||||
{
|
||||
printf '%s\n' "IMPORTANT: Do NOT read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. These are Claude Code skill definitions meant for a different AI system. Do not invoke any installed skill (Codex home skills/, .agents/); answer directly. Do NOT modify agents/openai.yaml. Stay focused on repository code only."
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# Model policy
|
||||
|
||||
Use a frontier model to challenge a plan, then a smart model to build it.
|
||||
These are curated recommendations, not a claim that one model wins every task.
|
||||
User choices take precedence, and a recommendation never switches the running
|
||||
session or edits the host's native configuration.
|
||||
|
||||
| Tier | Default job | Anthropic | OpenAI |
|
||||
| --- | --- | --- | --- |
|
||||
| `frontier` | Independent plan review | `claude-fable-5-1` | `gpt-6-astra` |
|
||||
| `smart` | Implementation recommendation | `claude-opus-5-5` | `gpt-6.1-sol` |
|
||||
|
||||
## Inspect and change your policy
|
||||
|
||||
This quickstart assumes gstack is installed. It uses no network, credentials or
|
||||
paid model calls. Run it from the project's repository root in Bash, Git Bash or MSYS:
|
||||
automated outside reviews use that directory. Inspection from a subdirectory
|
||||
reflects that subdirectory's native settings and may differ. If your install is vendored
|
||||
or elsewhere, set `GSTACK_ROOT` to that installed gstack runtime instead.
|
||||
|
||||
```bash
|
||||
if [ -x "$HOME/.claude/skills/gstack/bin/gstack-models" ]; then
|
||||
GSTACK_ROOT="$HOME/.claude/skills/gstack"
|
||||
elif [ -x "${CODEX_HOME:-$HOME/.codex}/skills/gstack/bin/gstack-models" ]; then
|
||||
GSTACK_ROOT="${CODEX_HOME:-$HOME/.codex}/skills/gstack"
|
||||
else
|
||||
printf '%s\n' 'Upgrade the gstack install for your host, then rerun this quickstart.' >&2
|
||||
exit 1
|
||||
fi
|
||||
"$GSTACK_ROOT/bin/gstack-models" list
|
||||
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai
|
||||
```
|
||||
|
||||
The output explains the selected model and which setting won. It also shows
|
||||
catalog verification metadata. Offline selection does **not** prove that a CLI
|
||||
is installed, that your account can use the model, or that a vendor's current
|
||||
catalog has been checked.
|
||||
|
||||
Choose the less expensive review tier, then inspect the effective result:
|
||||
|
||||
```bash
|
||||
"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier smart
|
||||
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai
|
||||
```
|
||||
|
||||
Pin a provider's model within a tier, or reset a pin to the shipped default:
|
||||
|
||||
```bash
|
||||
"$GSTACK_ROOT/bin/gstack-config" set model_frontier_openai gpt-6-astra
|
||||
"$GSTACK_ROOT/bin/gstack-config" set model_frontier_claude claude-fable-5-1
|
||||
"$GSTACK_ROOT/bin/gstack-config" unset model_frontier_openai
|
||||
"$GSTACK_ROOT/bin/gstack-config" unset plan_review_tier
|
||||
```
|
||||
|
||||
To keep using native CLI model settings for plan reviews:
|
||||
|
||||
```bash
|
||||
"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier host
|
||||
```
|
||||
|
||||
Settings apply to the next invocation without rerendering skills. An invocation
|
||||
already being checked keeps its selected model through execution. Configuration
|
||||
uses the existing [gstack state root](state-root.md), not a new per-repository
|
||||
settings file. Invalid writes are rejected and leave the previous value intact;
|
||||
use `unset`, not an empty value, to restore a default.
|
||||
|
||||
## Settings and precedence
|
||||
|
||||
| Key | Default | Accepted value |
|
||||
| --- | --- | --- |
|
||||
| `plan_review_tier` | `frontier` | `frontier`, `smart`, `host` |
|
||||
| `implementation_tier` | `smart` | `frontier`, `smart` |
|
||||
| `model_frontier_claude` | Catalog's Anthropic frontier model | Explicit Anthropic model identifier |
|
||||
| `model_frontier_openai` | Catalog's OpenAI frontier model | Explicit OpenAI model identifier |
|
||||
| `model_smart_claude` | Catalog's Anthropic smart model | Explicit Anthropic model identifier |
|
||||
| `model_smart_openai` | Catalog's OpenAI smart model | Explicit OpenAI model identifier |
|
||||
|
||||
For plan reviews, a model named for that invocation wins over
|
||||
`GSTACK_CLAUDE_MODEL` or `GSTACK_CODEX_MODEL`. That environment override wins over
|
||||
the selected tier's configured model, which wins over the shipped catalog.
|
||||
Generic coding-model settings do not silently select a cheaper plan reviewer;
|
||||
choose `host` mode when that is the behavior you want.
|
||||
|
||||
In `host` mode, Codex keeps its existing native-config resolution, including
|
||||
`CODEX_HOME`, active profiles and the `exec` versus native `review` distinction.
|
||||
Claude can delegate model selection to its CLI without forcing `--model`; the
|
||||
inspector reports that delegation rather than inventing an exact model ID.
|
||||
Actual returned model usage remains separate from the requested model.
|
||||
|
||||
Implementation recommendations use a model explicitly requested for that job,
|
||||
then the configured implementation tier and its provider override, then the
|
||||
catalog. Outside-review environment overrides do not change implementation
|
||||
recommendations. Known Claude Code and Codex hosts recommend their own provider;
|
||||
an unknown or multi-provider host can show both choices.
|
||||
|
||||
Custom endpoints and partner platforms need an explicitly compatible model or
|
||||
`host` mode. Gstack does not guess that public API IDs work with Bedrock, Vertex,
|
||||
Foundry or a custom Codex provider. A model identifier passing syntax validation
|
||||
is not proof of account availability. See [provider recovery](troubleshooting.md#model-policy-provider).
|
||||
|
||||
Detection is conservative, not a replacement for the CLI's settings engine.
|
||||
Claude checks environment, user settings, project/shared/local settings, local
|
||||
settings at the repository and worktree's main checkout, and local managed files.
|
||||
Codex checks environment, user/active-profile settings and system configuration.
|
||||
Any detected local custom-route signal requires an explicit model, even if
|
||||
native trust or precedence rules might suppress it. Server-managed or MDM
|
||||
policy and extra native CLI flags are not visible to this check; `--help` and
|
||||
JSON output list these limits. Use an explicit compatible model for those cases,
|
||||
or `host` mode for plan reviews. Implementation recommendations require an
|
||||
explicit implementation model rather than delegating to a hidden native choice.
|
||||
|
||||
## Which workflows use the tiers?
|
||||
|
||||
The plan-review role applies to autoplan's CEO, design, DX and engineering
|
||||
outside voices; standalone CEO, engineering, developer-experience and design
|
||||
plan reviews; and the spec quality gate. Existing reasoning effort, access
|
||||
restrictions, timeouts, review gates and `codex_reviews` consent remain in force.
|
||||
|
||||
The host selects the opposing provider independently from the tier. Claude Code
|
||||
uses the Codex wrapper; Codex uses the Claude Code wrapper. A model's name does
|
||||
not establish which harness is running.
|
||||
|
||||
Manual opposing-model wrappers can opt into `--role plan-review`. Ordinary
|
||||
code reviews, live design reviews, design consultation, office-hours second
|
||||
opinions and documentation reviews retain their existing model-selection
|
||||
behavior. Historical no-role defaults, eval actors/judges, cheap helper models
|
||||
and setup-time behavioral overlays do not follow this catalog.
|
||||
|
||||
Before the first affected paid plan review after upgrade, gstack explains the
|
||||
independent frontier default and the winning source, with `smart` and `host`
|
||||
alternatives. Inspection does not mark that notice as seen. Disabled outside
|
||||
reviews stay disabled; an unavailable model is reported, never silently replaced.
|
||||
|
||||
## First completed outside review
|
||||
|
||||
Policy setup and a completed review are different milestones. Install and
|
||||
authenticate the opposing CLI using the [host setup instructions](../README.md#install--about-30-seconds)
|
||||
before invoking a paid review. From Claude Code, use `/codex challenge --role
|
||||
plan-review` with your plan. From Codex, use `/claude-code challenge --role
|
||||
plan-review` (or its installed namespaced name).
|
||||
|
||||
Confirm the announced selection, the provider's actual result and the coverage
|
||||
status. A resolved model, a successful availability probe or a native fallback
|
||||
is not a completed outside review. An unusable explicit choice needs a repair
|
||||
at the source named in the error; changing a lower-priority tier cannot override
|
||||
an environment pin. See [selection recovery](troubleshooting.md#model-policy-selection).
|
||||
|
||||
## Machine-readable inspection
|
||||
|
||||
`gstack-models` with no arguments is `list`. Both `list` and `resolve` accept
|
||||
`--json`. Plan-review resolution requires `--provider anthropic|openai`;
|
||||
implementation resolution can omit the provider to return both recommendations.
|
||||
|
||||
```bash
|
||||
"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider anthropic --json
|
||||
"$GSTACK_ROOT/bin/gstack-models" resolve --role implementation --json
|
||||
```
|
||||
|
||||
JSON output is one document with `schemaVersion: 1`, a `selections` array and
|
||||
structured errors on failure. Human diagnostics go to stderr. Exit `0` means
|
||||
valid resolution, including host delegation; `1` means configuration or
|
||||
selection failure; `2` means invalid usage. Neither inspection nor resolution
|
||||
writes notice markers, probes a model or fetches vendor sources.
|
||||
|
||||
## Keeping shipped recommendations current
|
||||
|
||||
The model-policy freshness workflow runs weekly and supports manual dispatch.
|
||||
It checks official Anthropic and OpenAI recommendation and deprecation sources,
|
||||
without provider credentials or paid model calls. Normal PR checks use offline
|
||||
fixtures; a vendor outage does not make unrelated PRs fail.
|
||||
|
||||
In GitHub Actions, select **Model-policy freshness** to run it manually. From a
|
||||
gstack checkout, `bun scripts/model-policy-freshness.ts` performs the same public
|
||||
source check without changing an issue; unlike `gstack-models`, it uses the
|
||||
network. The workflow's guarded `--publish` path owns issue updates. Its JSON
|
||||
report and job summary retain the source evidence.
|
||||
|
||||
The report distinguishes `current`, `update-candidate`, `stale` and
|
||||
`unknown/source-unavailable`. Newer or newly recommended models are candidates,
|
||||
not automatic replacements. A support promise such as “not sooner than” is not
|
||||
a scheduled retirement, and an incidental model mention is not a new default.
|
||||
This evidence covers public API recommendations, not partner-platform entitlement.
|
||||
|
||||
One Actions-bot-owned tracking issue preserves the successful check's catalog/source/
|
||||
parser identity and any unresolved lifecycle findings. Healthy state remains
|
||||
readable when the issue is closed. A failed or partial check cannot erase a known
|
||||
retirement, borrow a success from an older catalog, or pretend its evidence is
|
||||
fresh. A catalog rollback reopens historical findings for a model put back in
|
||||
use. Only current default-branch runs may update that issue; copied markers in
|
||||
user-owned issues are not trusted state.
|
||||
|
||||
Thirty-day staleness means an absence of trustworthy freshness evidence, not
|
||||
that a model was released thirty days ago. An unchanged recommendation does not
|
||||
require a date-only catalog commit or a monthly paid benchmark. The recorded
|
||||
next-check due date helps maintenance, but a workflow that never runs cannot
|
||||
alert on its own absence.
|
||||
|
||||
### Maintainer update checklist
|
||||
|
||||
1. Check the latest freshness run and its due date. Resolve missing, corrupt or
|
||||
source-unavailable evidence before declaring the catalog current.
|
||||
2. Verify the official recommendation and lifecycle evidence. Triage announced
|
||||
retirement/removal within one working day, stale/unavailable evidence within
|
||||
seven days and discretionary update candidates within thirty days.
|
||||
3. Open a catalog change with the source URLs, exact model IDs and qualification
|
||||
evidence, or record why the current default should remain. Never edit users'
|
||||
personal pins or silently substitute another provider.
|
||||
4. Confirm CLI model/profile support and tool/reasoning compatibility. Keep the
|
||||
pinned test runtimes and benchmark rulers stable. A required CLI upgrade is a
|
||||
separate reviewed change, not an automatic side effect of discovery.
|
||||
5. Predeclare any budgeted baseline/candidate comparison using existing eval
|
||||
machinery, fixed prompts/panels and unchanged thresholds. Keep every attempt,
|
||||
measure useful and unsupported findings, latency and available usage, and do
|
||||
not treat vendor marketing or a creation timestamp as quality evidence.
|
||||
6. Pass the affected free checks and selected workflow evaluations, review the
|
||||
change, and ship it through the normal release process. Only unpinned
|
||||
role-bearing defaults follow the new shipped catalog.
|
||||
|
||||
The workflow does not open an automatic model-upgrade PR, merge changes, switch
|
||||
active sessions, alter benchmark models or schedule paid evaluations.
|
||||
@@ -22,6 +22,110 @@ section has a stable link anchor; the reason codes and anchors come from
|
||||
|
||||
---
|
||||
|
||||
## Model policy
|
||||
|
||||
The [model-policy guide](model-policy.md) explains the two tiers, all six
|
||||
settings, precedence and the read-only inspection command. A resolved selection
|
||||
is not proof that the requested provider or model can run.
|
||||
|
||||
<a id="model-policy-config"></a>
|
||||
### Model policy configuration is invalid or unreadable
|
||||
|
||||
**Meaning.** The new policy path could not read a coherent configuration or a
|
||||
configured value is invalid. An unreadable file is not treated as an absent
|
||||
setting: doing that could silently select the premium default.
|
||||
|
||||
**Fix.** Repair the file or key named by the error. `plan_review_tier` accepts
|
||||
`frontier`, `smart` or `host`; `implementation_tier` accepts `frontier` or
|
||||
`smart`. Use `gstack-config unset <key>` to restore a default, not an empty
|
||||
value. A rejected `set` leaves the old value intact. Correct ownership/read
|
||||
permissions rather than deleting an unrelated state directory.
|
||||
|
||||
**Expected result.** `gstack-models list` or the corresponding `resolve`
|
||||
command explains the winning source. No outside model call ran on this error.
|
||||
|
||||
<a id="model-policy-provider"></a>
|
||||
### A custom provider needs an explicit model or host mode
|
||||
|
||||
**Meaning.** Native settings or environment flags route the reviewer to a
|
||||
custom endpoint or partner platform. A public API catalog identifier is not
|
||||
necessarily a valid deployment identifier there. Malformed native provider
|
||||
configuration also prevents gstack from assuming the public endpoint.
|
||||
|
||||
**Fix.** Supply a compatible model for that request, set the provider's gstack
|
||||
environment override or tier-model setting, or choose
|
||||
`gstack-config set plan_review_tier host`. Keep credentials in the provider's
|
||||
normal authentication mechanism, never in a model setting. If the error names
|
||||
a malformed native configuration, fix that source first.
|
||||
|
||||
**Expected result.** Inspection reports an explicit selection or honest
|
||||
host-managed delegation. It does not certify account entitlement or send a
|
||||
public default to an inferred custom endpoint.
|
||||
|
||||
<a id="model-policy-selection"></a>
|
||||
### The selected review model could not be used
|
||||
|
||||
**Meaning.** The requested model was rejected or the provider could not
|
||||
complete the review. A model may be nonexistent, unavailable to the account,
|
||||
retired, or behind a misconfigured endpoint; preserve the provider's evidence
|
||||
rather than treating all of these as the same diagnosis. No successful outside
|
||||
review is implied, and gstack does not substitute a different model.
|
||||
|
||||
**Fix.** Change the source named in the error. For example, an environment
|
||||
override beats a tier setting, so editing the tier cannot replace it. Unset or
|
||||
correct that override, or explicitly name a supported model for the request.
|
||||
Follow the existing auth, quota, sandbox or timeout repair when that is the
|
||||
reported failure. Missing usage identity remains unknown, even if inspection
|
||||
showed a requested ID.
|
||||
|
||||
**Expected result.** The next invocation announces its model and source,
|
||||
checks that same selection and reports the actual provider result. A cached
|
||||
probe or a native fallback is not a completed outside review.
|
||||
|
||||
<a id="model-policy-freshness"></a>
|
||||
### Model-catalog freshness is unknown, stale or needs an update
|
||||
|
||||
**Meaning.** Official evidence changed, a required source could not be
|
||||
checked, or the retained evidence does not match this catalog and parser.
|
||||
`unknown/source-unavailable` is not current. A later source failure must not
|
||||
erase an earlier retirement finding or close its maintenance issue.
|
||||
|
||||
**Fix.** Read the failed source and evidence details in the freshness workflow
|
||||
report and tracking issue. Repair the source parser or publication access when
|
||||
needed, then run the workflow manually against the default branch. If a model
|
||||
was superseded or deprecated, follow the [maintainer checklist](model-policy.md#maintainer-update-checklist)
|
||||
to qualify a replacement or record a supported mitigation. Do not clear the
|
||||
issue just because a new release has not been adopted yet.
|
||||
|
||||
**Expected result.** Complete source checks produce evidence bound to the
|
||||
current catalog, while unresolved lifecycle findings remain visible. Personal
|
||||
pins and unrelated PRs are unchanged. Check the workflow's last run date too:
|
||||
a scheduled job that never runs cannot report its own absence.
|
||||
|
||||
#### Repairing `recoveryRequired` state
|
||||
|
||||
This flag deliberately keeps the issue open until a maintainer repairs the
|
||||
record. Another successful fetch alone cannot certify corrupted history.
|
||||
|
||||
1. Save the current issue body, including its retained recovery archive, and
|
||||
download the latest trustworthy workflow report. Use its `report.state` as
|
||||
the starting state; do not invent run IDs, catalog hashes or check dates.
|
||||
2. Reconcile its `lifecycle` array with the issue's independent lifecycle block
|
||||
and any retained archive. Preserve known retirements, deadlines and human
|
||||
dispositions. A model currently in the catalog must not keep an old
|
||||
`resolvedByCatalog` flag from a previous replacement.
|
||||
3. In the issue editor, restore valid JSON between the `state:begin` and
|
||||
`state:end` marker lines. Put that same lifecycle array between
|
||||
`lifecycle:begin` and `lifecycle:end`. Restore the standalone ownership and
|
||||
region markers, and leave human text outside the owned region intact.
|
||||
4. Set `recoveryRequired` to `false` only after that reconciliation. If prior
|
||||
successful evidence cannot be verified, set `lastSuccess` to `null` rather
|
||||
than making up a fresh success. If lifecycle history cannot be recovered,
|
||||
leave the flag set and the issue open for investigation.
|
||||
5. Run **Model-policy freshness** against the default branch. Only complete,
|
||||
current evidence with no unresolved findings can close the issue. A parsing
|
||||
or source error leaves it open; do not clear history to manufacture a pass.
|
||||
|
||||
## Outside reviews (Codex and Claude Code)
|
||||
|
||||
Outside reviews send your diff, plan or question to a second AI provider. Their
|
||||
|
||||
@@ -7,7 +7,12 @@ import { spawnSync } from 'node:child_process';
|
||||
const OLD = 'gstack-claude';
|
||||
const NEXT = 'gstack-claude-code';
|
||||
const BANNER = '<!-- AUTO-GENERATED from';
|
||||
const RUNTIME_FILES = ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts'];
|
||||
export const CLAUDE_CODE_RUNTIME_FILES = [
|
||||
'bin/gstack-claude-code', 'bin/gstack-models', 'bin/gstack-config', 'bin/gstack-state-root.sh',
|
||||
'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts',
|
||||
'lib/outside-review-result.ts', 'lib/gate-outcomes.ts', 'lib/state-root.ts',
|
||||
'lib/model-catalog.ts', 'lib/model-policy.ts', 'lib/model-policy-cli.ts', 'lib/model-policy-notice.ts',
|
||||
];
|
||||
|
||||
export interface RenameOptions {
|
||||
installDir: string;
|
||||
@@ -171,7 +176,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
|
||||
(fs.lstatSync(runtime).isSymbolicLink() && !linkIsOurs(runtime, root)) ||
|
||||
(fs.existsSync(path.join(runtime, 'SKILL.md')) && !generated(path.join(runtime, 'SKILL.md')))
|
||||
)) throw new Error(`runtime root is not gstack-managed: ${runtime}`);
|
||||
for (const rel of RUNTIME_FILES) {
|
||||
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
|
||||
if (!fs.existsSync(path.join(root, rel))) throw new Error(`replacement runtime is missing ${rel}`);
|
||||
const parent = path.join(runtime, path.dirname(rel));
|
||||
if (fs.lstatSync(parent, { throwIfNoEntry: false })?.isSymbolicLink() && !linkIsOurs(parent, root)) {
|
||||
@@ -190,7 +195,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
|
||||
throw new Error(`replacement render is not a managed directory: ${canonical}`);
|
||||
}
|
||||
copySkill(skill, canonical, root);
|
||||
for (const rel of RUNTIME_FILES) {
|
||||
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
|
||||
const src = path.join(root, rel);
|
||||
const dst = path.join(runtime, rel);
|
||||
if (fs.existsSync(dst) && fs.realpathSync(dst) === fs.realpathSync(src)) continue;
|
||||
@@ -206,7 +211,7 @@ export function migrateClaudeCodeSkills(opts: RenameOptions): { migrated: number
|
||||
if (!/^name:\s*(?:gstack-)?claude-code\s*$/m.test(fs.readFileSync(path.join(next, 'SKILL.md'), 'utf8'))) {
|
||||
throw new Error('installed replacement could not be verified');
|
||||
}
|
||||
for (const rel of RUNTIME_FILES) fs.accessSync(path.join(runtime, rel), fs.constants.R_OK);
|
||||
for (const rel of CLAUDE_CODE_RUNTIME_FILES) fs.accessSync(path.join(runtime, rel), fs.constants.R_OK);
|
||||
// Existing copied workflows must receive the same native host routing as
|
||||
// the new wrapper, including when setup selected a different host. Only
|
||||
// refresh installed managed entries; this never installs another host or
|
||||
|
||||
+54
-14
@@ -2,6 +2,8 @@
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { resolveClaudeCommand, type ClaudeCommand } from './claude-bin';
|
||||
import { initializeWindowsReviewJob, WindowsReviewSupervisionError } from './claude-code-windows-job';
|
||||
import { ModelPolicyError, describeSelection, resolvePlanReviewModel, type ModelSelection } from './model-policy';
|
||||
import { emitModelPolicyNotice } from './model-policy-notice';
|
||||
|
||||
export const CLAUDE_CODE_OUTPUT_LIMIT = 32 * 1024 * 1024;
|
||||
const DRAIN_TIMEOUT_MS = 500;
|
||||
@@ -13,6 +15,18 @@ export interface ClaudeCodeOptions {
|
||||
prompt: string;
|
||||
resume?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** Caller-owned workflow role; only plan-review exists. Absent keeps legacy model behavior. */
|
||||
role?: 'plan-review';
|
||||
/** A record already resolved for this invocation; it is used as-is and never re-resolved. */
|
||||
selection?: ModelSelection;
|
||||
}
|
||||
|
||||
export interface ClaudeCodeSelectionSummary {
|
||||
role: ModelSelection['role'];
|
||||
tier: ModelSelection['tier'];
|
||||
status: ModelSelection['status'];
|
||||
requested_model: string | null;
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface ClaudeCodeResult {
|
||||
@@ -24,6 +38,8 @@ export interface ClaudeCodeResult {
|
||||
usage?: Record<string, unknown>;
|
||||
modelUsage?: Record<string, unknown>;
|
||||
model?: string;
|
||||
/** What gstack requested, separate from the CLI's reported model/modelUsage. */
|
||||
selection?: ClaudeCodeSelectionSummary;
|
||||
exit_code?: number | null;
|
||||
stderr?: string;
|
||||
}
|
||||
@@ -37,7 +53,7 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
||||
}
|
||||
|
||||
/** Keep argument construction separate so wrappers never interpolate a prompt. */
|
||||
export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resume'>, command: ClaudeCommand, env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resume' | 'selection'>, command: ClaudeCommand, env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
const args = [
|
||||
...command.argsPrefix, '-p', '--output-format', 'json',
|
||||
'--disable-slash-commands',
|
||||
@@ -58,9 +74,12 @@ export function claudeCodeArgs(options: Pick<ClaudeCodeOptions, 'access' | 'resu
|
||||
+ 'If essential context is missing, identify it explicitly instead of inventing observations.');
|
||||
}
|
||||
if (options.access === 'read-only') args.push('--allowedTools', 'Read,Grep,Glob');
|
||||
// An explicit gstack override wins; otherwise leave native CLI configuration
|
||||
// and ANTHROPIC_MODEL intact instead of replacing the user's selected model.
|
||||
if (env.GSTACK_CLAUDE_MODEL) args.push('--model', env.GSTACK_CLAUDE_MODEL);
|
||||
// A selected record already ranked GSTACK_CLAUDE_MODEL; delegated host mode
|
||||
// passes no model. One joined argument keeps an ID starting with '-' a value. Without a record, an explicit gstack override wins;
|
||||
// otherwise leave native CLI configuration and ANTHROPIC_MODEL intact.
|
||||
if (options.selection) {
|
||||
if (options.selection.status === 'selected') args.push(`--model=${options.selection.requestedModel}`);
|
||||
} else if (env.GSTACK_CLAUDE_MODEL) args.push('--model', env.GSTACK_CLAUDE_MODEL);
|
||||
if (options.resume) args.push('--resume', options.resume);
|
||||
return args;
|
||||
}
|
||||
@@ -127,24 +146,43 @@ export function parseClaudeCodeResult(stdout: string, stderr: string, exitCode:
|
||||
export async function runClaudeCode(options: ClaudeCodeOptions): Promise<ClaudeCodeResult> {
|
||||
if (!options.cwd || !['none', 'read-only'].includes(options.access) ||
|
||||
!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0 || options.timeoutMs > 2_147_483_647 ||
|
||||
!options.prompt.trim() || (options.resume !== undefined && !options.resume.trim())) {
|
||||
!options.prompt.trim() || (options.resume !== undefined && !options.resume.trim()) ||
|
||||
(options.role !== undefined && options.role !== 'plan-review')) {
|
||||
return failure('arguments', 'Provide --cwd, --access none|read-only, a positive --timeout-ms, and a nonempty prompt on stdin.');
|
||||
}
|
||||
const env = options.env ?? process.env;
|
||||
const command = resolveClaudeCommand(env);
|
||||
if (!command) return failure('not-found', 'Claude Code CLI not found. Install Claude Code or set GSTACK_CLAUDE_BIN, then retry.', true);
|
||||
let selection = options.selection;
|
||||
if (!selection && options.role === 'plan-review') {
|
||||
try {
|
||||
selection = resolvePlanReviewModel({ provider: 'anthropic', env, cwd: options.cwd });
|
||||
} catch (error) {
|
||||
if (!(error instanceof ModelPolicyError)) throw error;
|
||||
return failure(error.reason, `${error.problem} ${error.cause} Repair: ${error.repair.join('; ')}. ${error.docs} No Claude Code call was made; outside coverage is unavailable.`, true);
|
||||
}
|
||||
}
|
||||
if (selection) {
|
||||
process.stderr.write(`CLAUDE_MODEL: ${describeSelection(selection)}\n`);
|
||||
if (selection.role === 'plan-review') emitModelPolicyNotice(selection, { env });
|
||||
}
|
||||
const summary: ClaudeCodeSelectionSummary | undefined = selection && {
|
||||
role: selection.role, tier: selection.tier, status: selection.status,
|
||||
requested_model: selection.requestedModel, source: selection.source.label,
|
||||
};
|
||||
const withSelection = (result: ClaudeCodeResult): ClaudeCodeResult => (summary ? { ...result, selection: summary } : result);
|
||||
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = spawn(command.command, claudeCodeArgs(options, command, env), {
|
||||
child = spawn(command.command, claudeCodeArgs({ ...options, selection }, command, env), {
|
||||
cwd: options.cwd, env, stdio: ['pipe', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32', windowsHide: true,
|
||||
});
|
||||
} catch (error) {
|
||||
return failure('spawn', `Claude Code could not start: ${(error as Error).message}`, true);
|
||||
return withSelection(failure('spawn', `Claude Code could not start: ${(error as Error).message}`, true));
|
||||
}
|
||||
|
||||
return await new Promise<ClaudeCodeResult>((resolve) => {
|
||||
return withSelection(await new Promise<ClaudeCodeResult>((resolve) => {
|
||||
let settled = false;
|
||||
let stopped: ClaudeCodeResult | undefined;
|
||||
let exitCode: number | null = null;
|
||||
@@ -222,7 +260,7 @@ export async function runClaudeCode(options: ClaudeCodeOptions): Promise<ClaudeC
|
||||
}
|
||||
});
|
||||
child.stdin?.end(options.prompt);
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
export async function claudeCodeMain(argv: string[]): Promise<number> {
|
||||
@@ -230,22 +268,24 @@ export async function claudeCodeMain(argv: string[]): Promise<number> {
|
||||
try {
|
||||
const values = new Map<string, string>();
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
if (!['--cwd', '--access', '--timeout-ms', '--resume'].includes(argv[i]) || values.has(argv[i]) || argv[i + 1] === undefined) {
|
||||
throw new Error('Usage: gstack-claude-code --cwd <repo> --access none|read-only --timeout-ms <n> [--resume <session-id>] (prompt on stdin)');
|
||||
if (!['--cwd', '--access', '--timeout-ms', '--resume', '--role'].includes(argv[i]) || values.has(argv[i]) || argv[i + 1] === undefined) {
|
||||
throw new Error('Usage: gstack-claude-code --cwd <repo> --access none|read-only --timeout-ms <n> [--resume <session-id>] [--role plan-review] (prompt on stdin)');
|
||||
}
|
||||
values.set(argv[i], argv[i + 1]);
|
||||
}
|
||||
const cwd = values.get('--cwd') ?? '';
|
||||
const access = values.get('--access') as ClaudeCodeOptions['access'];
|
||||
const timeoutMs = Number(values.get('--timeout-ms'));
|
||||
if (!cwd || !['none', 'read-only'].includes(access) || !Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647) {
|
||||
throw new Error('Provide --cwd, --access none|read-only, and a positive --timeout-ms.');
|
||||
const role = values.get('--role');
|
||||
if (!cwd || !['none', 'read-only'].includes(access) || !Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647 ||
|
||||
(role !== undefined && role !== 'plan-review')) {
|
||||
throw new Error('Provide --cwd, --access none|read-only, a positive --timeout-ms, and optionally --role plan-review.');
|
||||
}
|
||||
// This entry runs in a dedicated process. Its Windows job owns the CLI and
|
||||
// descendants even after the immediate provider process exits; the final
|
||||
// process.exit happens only after the result below has flushed to stdout.
|
||||
await initializeWindowsReviewJob();
|
||||
result = await runClaudeCode({ cwd, access, timeoutMs, resume: values.get('--resume'), prompt: await Bun.stdin.text() });
|
||||
result = await runClaudeCode({ cwd, access, timeoutMs, resume: values.get('--resume'), role: role as ClaudeCodeOptions['role'], prompt: await Bun.stdin.text() });
|
||||
} catch (error) {
|
||||
result = error instanceof WindowsReviewSupervisionError
|
||||
? failure('supervision', error.message, true)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export type ModelProvider = 'anthropic' | 'openai';
|
||||
export type ModelTier = 'frontier' | 'smart';
|
||||
|
||||
export const MODEL_PROVIDERS: readonly ModelProvider[] = Object.freeze(['anthropic', 'openai'] as const);
|
||||
export const MODEL_TIERS: readonly ModelTier[] = Object.freeze(['frontier', 'smart'] as const);
|
||||
export const MODEL_CATALOG_VERSION = 1 as const;
|
||||
|
||||
export interface ModelCatalogEntry {
|
||||
readonly provider: ModelProvider;
|
||||
readonly tier: ModelTier;
|
||||
readonly model: string;
|
||||
readonly sourceUrl: string;
|
||||
readonly verifiedAt: string;
|
||||
}
|
||||
|
||||
const ANTHROPIC_SOURCE = 'https://platform.claude.com/docs/en/models/overview';
|
||||
const OPENAI_SOURCE = 'https://developers.openai.com/api/docs/models';
|
||||
|
||||
export const MODEL_CATALOG: readonly ModelCatalogEntry[] = Object.freeze([
|
||||
{ provider: 'anthropic', tier: 'frontier', model: 'claude-fable-5-1', sourceUrl: ANTHROPIC_SOURCE, verifiedAt: '2026-10-07' },
|
||||
{ provider: 'openai', tier: 'frontier', model: 'gpt-6-astra', sourceUrl: OPENAI_SOURCE, verifiedAt: '2026-10-07' },
|
||||
{ provider: 'anthropic', tier: 'smart', model: 'claude-opus-5-5', sourceUrl: ANTHROPIC_SOURCE, verifiedAt: '2026-10-07' },
|
||||
{ provider: 'openai', tier: 'smart', model: 'gpt-6.1-sol', sourceUrl: OPENAI_SOURCE, verifiedAt: '2026-10-07' },
|
||||
].map(entry => Object.freeze(entry as ModelCatalogEntry)));
|
||||
|
||||
export function catalogEntry(provider: ModelProvider, tier: ModelTier): ModelCatalogEntry {
|
||||
const entry = MODEL_CATALOG.find(e => e.provider === provider && e.tier === tier);
|
||||
if (!entry) throw new Error(`model catalog has no ${tier} entry for ${provider}`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function modelCatalogCanonicalJson(): string {
|
||||
return JSON.stringify({
|
||||
version: MODEL_CATALOG_VERSION,
|
||||
entries: MODEL_CATALOG.map(e => ({ provider: e.provider, tier: e.tier, model: e.model, sourceUrl: e.sourceUrl, verifiedAt: e.verifiedAt })),
|
||||
});
|
||||
}
|
||||
|
||||
export function modelCatalogSha256(): string {
|
||||
return createHash('sha256').update(modelCatalogCanonicalJson()).digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
import * as path from 'node:path';
|
||||
import { MODEL_CATALOG, MODEL_CATALOG_VERSION, MODEL_PROVIDERS, MODEL_TIERS, modelCatalogSha256, type ModelProvider } from './model-catalog';
|
||||
import {
|
||||
CUSTOM_PROVIDER_UNSCANNED,
|
||||
describeSelection,
|
||||
MODEL_OVERRIDE_KEYS,
|
||||
ModelPolicyError,
|
||||
modelPolicyCommands,
|
||||
readModelPolicyConfig,
|
||||
resolveImplementationModels,
|
||||
resolvePlanReviewModel,
|
||||
type ModelPolicyConfig,
|
||||
type ModelPolicyErrorJson,
|
||||
type ModelRole,
|
||||
type ModelSelection,
|
||||
} from './model-policy';
|
||||
import type { StateRootEnv } from './state-root';
|
||||
|
||||
export interface ModelsCliResult {
|
||||
code: 0 | 1 | 2;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
type CliError = (ModelPolicyErrorJson | { reason: 'usage'; problem: string; cause: string; key: null; source: null; repair: string[]; docs: string }) & { role?: ModelRole; provider?: ModelProvider };
|
||||
|
||||
const OFFLINE_NOTE = 'offline: no network, no paid model call and no writes; this does not prove account entitlement, CLI availability or current vendor status';
|
||||
const DOCS = 'https://github.com/garrytan/gstack/blob/main/docs/model-policy.md';
|
||||
const CONFIG_BIN = path.join(import.meta.dir, '..', 'bin', 'gstack-config');
|
||||
|
||||
export const MODELS_HELP = `Usage:
|
||||
gstack-models [list] [--json]
|
||||
gstack-models resolve --role plan-review --provider anthropic|openai [--json]
|
||||
gstack-models resolve --role implementation [--provider anthropic|openai] [--json]
|
||||
gstack-models --help
|
||||
|
||||
Shows which model each gstack role uses and why. It reads config.yaml, the
|
||||
GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL environment variables and native
|
||||
provider settings, then exits. It is offline: no network, no paid model call,
|
||||
no writes. A resolved model is not proof that your account can use it, that
|
||||
the CLI is installed, or that the vendor still offers it.
|
||||
|
||||
Roles:
|
||||
plan-review The opposing-provider reviewer for plan reviews. Default tier
|
||||
frontier. Precedence: explicit request, then
|
||||
GSTACK_CLAUDE_MODEL (anthropic) or GSTACK_CODEX_MODEL (openai),
|
||||
then model_<tier>_<claude|openai>, then the shipped catalog.
|
||||
plan_review_tier=host delegates to the host CLI's own settings:
|
||||
the model is host-controlled and unknown until the review runs.
|
||||
implementation The model gstack recommends for implementing an approved plan.
|
||||
Default tier smart. Precedence: explicit request, then
|
||||
model_<tier>_<claude|openai>, then the shipped catalog.
|
||||
Environment overrides do not apply. Without --provider both
|
||||
provider recommendations are shown. gstack never switches your
|
||||
session's model.
|
||||
|
||||
Settings (gstack-config set|unset <key>):
|
||||
plan_review_tier frontier | smart | host (default frontier)
|
||||
implementation_tier frontier | smart (default smart)
|
||||
model_frontier_claude, model_frontier_openai,
|
||||
model_smart_claude, model_smart_openai (default: shipped catalog ID)
|
||||
|
||||
A custom Anthropic or OpenAI endpoint (Bedrock, Vertex, Foundry, a base-URL
|
||||
override or a Codex model_provider) never receives a shipped public ID: set a
|
||||
per-tier model, an environment override (plan-review only) or host mode.
|
||||
Detection reads the environment, the env block of Claude's user, project
|
||||
(.claude/settings.json, .claude/settings.local.json in the working directory
|
||||
and repository root) and managed settings files, and Codex's user and system
|
||||
config.toml. It cannot see server-managed or MDM policy, claude --settings, or
|
||||
codex -c/--profile flags: if you route those to a custom endpoint, set a
|
||||
per-tier model or host mode.
|
||||
|
||||
Output: --json prints one JSON document (schemaVersion 1) with a selections
|
||||
array and structured errors. Human diagnostics go to stderr.
|
||||
Exit: 0 valid resolution (including host delegation), 1 config or selection
|
||||
failure, 2 invalid usage.
|
||||
Docs: ${DOCS}
|
||||
`;
|
||||
|
||||
function usage(problem: string, json: boolean): ModelsCliResult {
|
||||
const error: CliError = { reason: 'usage', problem, cause: 'see gstack-models --help', key: null, source: null, repair: ['gstack-models --help'], docs: DOCS };
|
||||
if (json) return { code: 2, stdout: `${JSON.stringify(document(null, null, [], [error]), null, 2)}\n`, stderr: '' };
|
||||
return { code: 2, stdout: '', stderr: `gstack-models: ${problem}\n${MODELS_HELP.split('\n\n')[0]}\n` };
|
||||
}
|
||||
|
||||
function parseProvider(value: string | undefined): ModelProvider | null {
|
||||
return value === 'anthropic' || value === 'openai' ? value : null;
|
||||
}
|
||||
|
||||
function document(command: 'list' | 'resolve' | null, config: ModelPolicyConfig | null, selections: ModelSelection[], errors: CliError[]) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
command,
|
||||
ok: errors.length === 0,
|
||||
offline: true,
|
||||
note: OFFLINE_NOTE,
|
||||
configPath: config?.path ?? null,
|
||||
catalog: { version: MODEL_CATALOG_VERSION, sha256: modelCatalogSha256(), entries: MODEL_CATALOG },
|
||||
settings: config ? settingsOf(config) : null,
|
||||
tiers: config ? tiersOf(config) : null,
|
||||
providerDetection: { scope: 'declared local settings files and environment only', unscanned: CUSTOM_PROVIDER_UNSCANNED },
|
||||
selections,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
function settingsOf(config: ModelPolicyConfig) {
|
||||
const settings: Record<string, { value: string | null; effective: string; origin: 'config' | 'default' }> = {
|
||||
plan_review_tier: { value: config.planReviewTier.origin === 'config' ? config.planReviewTier.value : null, effective: config.planReviewTier.value, origin: config.planReviewTier.origin },
|
||||
implementation_tier: { value: config.implementationTier.origin === 'config' ? config.implementationTier.value : null, effective: config.implementationTier.value, origin: config.implementationTier.origin },
|
||||
};
|
||||
for (const entry of MODEL_CATALOG) {
|
||||
const override = config.overrides[entry.provider][entry.tier];
|
||||
settings[MODEL_OVERRIDE_KEYS[entry.provider][entry.tier]] = { value: override, effective: override ?? entry.model, origin: override ? 'config' : 'default' };
|
||||
}
|
||||
return settings;
|
||||
}
|
||||
|
||||
function tiersOf(config: ModelPolicyConfig) {
|
||||
return MODEL_CATALOG.map(entry => ({
|
||||
tier: entry.tier,
|
||||
provider: entry.provider,
|
||||
catalogModel: entry.model,
|
||||
verifiedAt: entry.verifiedAt,
|
||||
sourceUrl: entry.sourceUrl,
|
||||
overrideKey: MODEL_OVERRIDE_KEYS[entry.provider][entry.tier],
|
||||
override: config.overrides[entry.provider][entry.tier],
|
||||
configuredModel: config.overrides[entry.provider][entry.tier] ?? entry.model,
|
||||
}));
|
||||
}
|
||||
|
||||
function attempt(role: ModelRole, provider: ModelProvider, run: () => ModelSelection[], selections: ModelSelection[], errors: CliError[]): void {
|
||||
try {
|
||||
selections.push(...run());
|
||||
} catch (error) {
|
||||
if (!(error instanceof ModelPolicyError)) throw error;
|
||||
errors.push({ ...error.toJSON(), role, provider });
|
||||
}
|
||||
}
|
||||
|
||||
function renderError(error: CliError): string {
|
||||
const scope = error.role ? ` (${error.role}${error.provider ? ` via ${error.provider}` : ''})` : '';
|
||||
return [
|
||||
`gstack-models: ${error.problem}${scope}`,
|
||||
` cause: ${error.cause}`,
|
||||
...(error.key ? [` source: ${error.key}`] : []),
|
||||
...error.repair.map(step => ` fix: ${step}`),
|
||||
` docs: ${error.docs}`,
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
|
||||
function renderSelections(role: ModelRole, config: ModelPolicyConfig, selections: ModelSelection[]): string[] {
|
||||
const choice = role === 'plan-review' ? config.planReviewTier : config.implementationTier;
|
||||
const purpose = role === 'plan-review' ? 'opposing-provider plan evaluation' : 'recommendation only; gstack never switches your session';
|
||||
const lines = [`${role}: tier ${choice.value} (${choice.origin === 'config' ? `${choice.key} in ${choice.path}` : `default; set ${choice.key}`}) - ${purpose}`];
|
||||
for (const selection of selections.filter(s => s.role === role)) {
|
||||
lines.push(` ${describeSelection(selection)}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function renderList(config: ModelPolicyConfig, selections: ModelSelection[]): string {
|
||||
const commands = modelPolicyCommands();
|
||||
const lines = [
|
||||
`gstack model policy (${OFFLINE_NOTE})`,
|
||||
`config: ${config.path}${config.exists ? '' : ' (not created yet; defaults apply)'}`,
|
||||
'',
|
||||
...renderSelections('plan-review', config, selections),
|
||||
...renderSelections('implementation', config, selections),
|
||||
'',
|
||||
`catalog v${MODEL_CATALOG_VERSION} (sha256 ${modelCatalogSha256().slice(0, 12)})`,
|
||||
];
|
||||
for (const tier of MODEL_TIERS) {
|
||||
for (const provider of MODEL_PROVIDERS) {
|
||||
const entry = MODEL_CATALOG.find(e => e.tier === tier && e.provider === provider)!;
|
||||
const override = config.overrides[provider][tier];
|
||||
lines.push(` ${tier.padEnd(8)} ${provider.padEnd(9)} ${entry.model.padEnd(18)} verified ${entry.verifiedAt} ${MODEL_OVERRIDE_KEYS[provider][tier]}: ${override ?? 'unset'}`);
|
||||
}
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'change (gstack-config is ' + CONFIG_BIN + '):',
|
||||
` ${commands.useSmart}`,
|
||||
` ${commands.useHost}`,
|
||||
` ${commands.restoreDefault}`,
|
||||
` ${commands.overrideExample('openai', 'frontier')}`,
|
||||
` ${commands.unsetOverride('openai', 'frontier')}`,
|
||||
'environment overrides GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL outrank plan-review tier settings',
|
||||
'custom-endpoint detection reads local settings files only; server-managed/MDM policy and CLI flags are not visible (see --help)',
|
||||
);
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function renderResolve(role: ModelRole, config: ModelPolicyConfig, selections: ModelSelection[]): string {
|
||||
const lines = renderSelections(role, config, selections);
|
||||
if (selections.some(s => s.status === 'delegated-host')) {
|
||||
lines.push(' host-controlled: gstack passes no model; the actual model is reported after the review runs');
|
||||
}
|
||||
lines.push(`(${OFFLINE_NOTE})`);
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
export function modelsMain(args: string[], env: StateRootEnv = process.env, platform: NodeJS.Platform = process.platform, cwd: string = process.cwd()): ModelsCliResult {
|
||||
const json = args.includes('--json');
|
||||
if (args.includes('--help') || args.includes('-h')) return { code: 0, stdout: MODELS_HELP, stderr: '' };
|
||||
const rest = [...args];
|
||||
const command = rest[0] === undefined || rest[0].startsWith('--') ? 'list' : rest.shift()!;
|
||||
if (command !== 'list' && command !== 'resolve') return usage(`unknown command '${command}'`, json);
|
||||
const options: Record<string, string> = {};
|
||||
while (rest.length) {
|
||||
const arg = rest.shift()!;
|
||||
if (arg === '--json') continue;
|
||||
const [name, inline] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, undefined];
|
||||
if (command === 'list' || (name !== '--role' && name !== '--provider')) return usage(`unknown argument '${arg}' for ${command}`, json);
|
||||
if (options[name] !== undefined) return usage(`${name} given twice`, json);
|
||||
const value = inline ?? rest.shift();
|
||||
if (value === undefined || value === '' || value.startsWith('--')) return usage(`${name} needs a value`, json);
|
||||
options[name] = value;
|
||||
}
|
||||
const role = options['--role'];
|
||||
const providerArg = options['--provider'];
|
||||
const provider = parseProvider(providerArg);
|
||||
if (command === 'resolve') {
|
||||
if (role !== 'plan-review' && role !== 'implementation') return usage(role === undefined ? 'resolve needs --role plan-review|implementation' : `unknown role '${role}': use plan-review or implementation`, json);
|
||||
if (providerArg !== undefined && !provider) return usage(`unknown provider '${providerArg}': use anthropic or openai`, json);
|
||||
if (role === 'plan-review' && !provider) return usage('resolve --role plan-review needs --provider anthropic|openai (the opposing provider for your host)', json);
|
||||
}
|
||||
|
||||
let config: ModelPolicyConfig;
|
||||
try {
|
||||
config = readModelPolicyConfig({ env, platform });
|
||||
} catch (error) {
|
||||
if (!(error instanceof ModelPolicyError)) throw error;
|
||||
const errors: CliError[] = [error.toJSON()];
|
||||
return json
|
||||
? { code: 1, stdout: `${JSON.stringify(document(command, null, [], errors), null, 2)}\n`, stderr: '' }
|
||||
: { code: 1, stdout: '', stderr: errors.map(renderError).join('') };
|
||||
}
|
||||
|
||||
const selections: ModelSelection[] = [];
|
||||
const errors: CliError[] = [];
|
||||
const roles: ModelRole[] = command === 'list' ? ['plan-review', 'implementation'] : [role as ModelRole];
|
||||
for (const r of roles) {
|
||||
for (const p of provider ? [provider] : MODEL_PROVIDERS) {
|
||||
attempt(r, p, () => r === 'plan-review'
|
||||
? [resolvePlanReviewModel({ provider: p, env, platform, cwd, config })]
|
||||
: resolveImplementationModels({ provider: p, env, platform, cwd, config }), selections, errors);
|
||||
}
|
||||
}
|
||||
const code = errors.length ? 1 : 0;
|
||||
if (json) return { code, stdout: `${JSON.stringify(document(command, config, selections, errors), null, 2)}\n`, stderr: '' };
|
||||
const stdout = command === 'list' ? renderList(config, selections) : renderResolve(role as ModelRole, config, selections);
|
||||
return { code, stdout, stderr: errors.map(renderError).join('') };
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { readBoundedResponse, sha256, type Fetcher } from './model-policy-freshness-sources';
|
||||
import { FRESHNESS_ISSUE_AUTHOR, type FreshnessGitHub, type TrackingIssue } from './model-policy-freshness-publication';
|
||||
|
||||
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
|
||||
|
||||
function issueFromJson(value: unknown): TrackingIssue {
|
||||
if (!object(value) || !Number.isSafeInteger(value.number) || Number(value.number) <= 0 || (value.body !== null && typeof value.body !== 'string') || !['open', 'closed'].includes(String(value.state)) || !object(value.user) || typeof value.user.login !== 'string') throw new Error('github-invalid-issue');
|
||||
return { number: Number(value.number), body: value.body === null ? '' : value.body as string, state: value.state as 'open' | 'closed', author: value.user.login };
|
||||
}
|
||||
|
||||
export function freshnessGitHub(repository: string, token: string, fetcher: Fetcher = fetch): FreshnessGitHub {
|
||||
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository) || !token) throw new Error('github-publication-prerequisites-missing');
|
||||
const root = `https://api.github.com/repos/${repository}`;
|
||||
async function request(path: string, signal: AbortSignal, method = 'GET', body?: unknown): Promise<unknown> {
|
||||
if (signal.aborted) throw new Error('github-publication-aborted');
|
||||
const response = await fetcher(`${root}${path}`, {
|
||||
method, signal, redirect: 'error',
|
||||
headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'Content-Type': 'application/json' },
|
||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||
});
|
||||
if (!response.ok) { await response.body?.cancel(); throw new Error(`github-http-${response.status}`); }
|
||||
try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(await readBoundedResponse(response))); }
|
||||
catch { throw new Error('github-invalid-or-oversized-response'); }
|
||||
}
|
||||
return {
|
||||
async defaultCatalog(signal) {
|
||||
const repo = await request('', signal);
|
||||
if (!object(repo) || typeof repo.default_branch !== 'string' || !repo.default_branch) throw new Error('github-default-branch-unavailable');
|
||||
const file = await request(`/contents/lib/model-catalog.ts?ref=${encodeURIComponent(repo.default_branch)}`, signal);
|
||||
if (!object(file) || file.encoding !== 'base64' || typeof file.content !== 'string' || typeof file.size !== 'number' || file.size <= 0 || file.size > 2 * 1024 * 1024 || !/^[A-Za-z0-9+/=\n\r]+$/.test(file.content)) throw new Error('github-default-catalog-unavailable');
|
||||
const bytes = Buffer.from(file.content, 'base64');
|
||||
if (bytes.length !== file.size) throw new Error('github-default-catalog-size-mismatch');
|
||||
return { defaultBranch: repo.default_branch, sourceSha256: sha256(bytes) };
|
||||
},
|
||||
async listIssues(signal) {
|
||||
const issues: TrackingIssue[] = [];
|
||||
for (let page = 1; page <= 100; page++) {
|
||||
const result = await request(`/issues?state=all&creator=${encodeURIComponent(FRESHNESS_ISSUE_AUTHOR)}&per_page=100&page=${page}`, signal);
|
||||
if (!Array.isArray(result)) throw new Error('github-issue-list-unavailable');
|
||||
for (const value of result) {
|
||||
const issue = issueFromJson(value);
|
||||
if (issue.author !== FRESHNESS_ISSUE_AUTHOR) throw new Error('github-issue-owner-filter-mismatch');
|
||||
if (object(value) && !value.pull_request) issues.push(issue);
|
||||
}
|
||||
if (result.length < 100) return issues;
|
||||
}
|
||||
throw new Error('github-issue-pagination-limit');
|
||||
},
|
||||
async readIssue(number, signal) { return issueFromJson(await request(`/issues/${number}`, signal)); },
|
||||
async createIssue(body, signal) { return issueFromJson(await request('/issues', signal, 'POST', { title: 'Model-policy freshness: advisory maintenance', body })); },
|
||||
async updateIssue(number, update, signal) { await request(`/issues/${number}`, signal, 'PATCH', update); },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { parseOfficialSource, type SourceId } from './model-policy-freshness-sources';
|
||||
|
||||
self.onmessage = (event: MessageEvent<{ id: SourceId; text: string }>) => {
|
||||
try { self.postMessage({ ok: true, parsed: parseOfficialSource(event.data.id, event.data.text) }); }
|
||||
catch (error) { self.postMessage({ ok: false, error: error instanceof Error ? error.message : 'parser-unavailable' }); }
|
||||
};
|
||||
@@ -0,0 +1,127 @@
|
||||
import { FRESHNESS_BOUNDS, withinDeadline } from './model-policy-freshness-sources';
|
||||
import { assessFreshness, observationIsOlder, renderFreshnessReport, sameCatalog, validLifecycle, validState, type CatalogIdentity, type FreshnessObservation, type FreshnessReport, type FreshnessState, type LifecycleFinding } from './model-policy-freshness';
|
||||
|
||||
export const FRESHNESS_ISSUE_MARKER = '<!-- gstack:model-policy-freshness:v1 -->';
|
||||
export const FRESHNESS_REGION_BEGIN = '<!-- gstack:model-policy-freshness:begin -->';
|
||||
export const FRESHNESS_REGION_END = '<!-- gstack:model-policy-freshness:end -->';
|
||||
const STATE_BEGIN = '<!-- gstack:model-policy-freshness:state:begin -->';
|
||||
const STATE_END = '<!-- gstack:model-policy-freshness:state:end -->';
|
||||
const HISTORY_BEGIN = '<!-- gstack:model-policy-freshness:lifecycle:begin -->';
|
||||
const HISTORY_END = '<!-- gstack:model-policy-freshness:lifecycle:end -->';
|
||||
export const FRESHNESS_ISSUE_AUTHOR = 'github-actions[bot]';
|
||||
export type TrackingIssue = { number: number; body: string; state: 'open' | 'closed'; author: string };
|
||||
export type FreshnessGitHub = {
|
||||
defaultCatalog: (signal: AbortSignal) => Promise<{ defaultBranch: string; sourceSha256: string }>;
|
||||
listIssues: (signal: AbortSignal) => Promise<TrackingIssue[]>;
|
||||
readIssue: (number: number, signal: AbortSignal) => Promise<TrackingIssue>;
|
||||
createIssue: (body: string, signal: AbortSignal) => Promise<TrackingIssue>;
|
||||
updateIssue: (number: number, update: { body: string; state: 'open' | 'closed' }, signal: AbortSignal) => Promise<void>;
|
||||
};
|
||||
export type StoredIssue = { state: FreshnessState | null; lifecycle: LifecycleFinding[]; recoveryRequired: boolean; problem: string | null };
|
||||
const KNOWN_MARKERS = [FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, STATE_BEGIN, STATE_END, HISTORY_BEGIN, HISTORY_END];
|
||||
|
||||
function markerLines(body: string, marker?: string): { start: number; end: number; text: string }[] {
|
||||
return [...body.matchAll(/[^\r\n]+/g)].flatMap(line => {
|
||||
const text = line[0];
|
||||
const matches = marker === undefined ? /^[\uFEFF \t]*(?:<!--|<!-)[ \t]*gstack:model-policy-freshness/i.test(text) : text === marker;
|
||||
return matches ? [{ start: line.index, end: line.index + text.length, text }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
const occurrences = (body: string, marker: string): number => markerLines(body, marker).length;
|
||||
const malformedMarkers = (body: string): boolean => markerLines(body).some(line => !KNOWN_MARKERS.includes(line.text));
|
||||
|
||||
function block(body: string, begin: string, end: string): { start: number; end: number; text: string } | null {
|
||||
const starts = markerLines(body, begin);
|
||||
const ends = markerLines(body, end);
|
||||
if (starts.length !== 1 || ends.length !== 1 || ends[0].start <= starts[0].end) return null;
|
||||
return { start: starts[0].start, end: ends[0].end, text: body.slice(starts[0].end, ends[0].start).trim() };
|
||||
}
|
||||
|
||||
function jsonBlock(body: string, begin: string, end: string): unknown {
|
||||
const text = block(body, begin, end)?.text;
|
||||
if (!text) return null;
|
||||
try { return JSON.parse(text); } catch { return null; }
|
||||
}
|
||||
|
||||
export function readFreshnessIssue(body: string): StoredIssue {
|
||||
const owned = block(body, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END);
|
||||
const history = jsonBlock(owned?.text ?? body, HISTORY_BEGIN, HISTORY_END);
|
||||
const state = jsonBlock(owned?.text ?? body, STATE_BEGIN, STATE_END);
|
||||
const lifecycle = validLifecycle(history) ? history : validState(state) ? state.lifecycle : [];
|
||||
if (owned && occurrences(body, FRESHNESS_ISSUE_MARKER) === 1 && !malformedMarkers(body) && validState(state) && validLifecycle(history) && JSON.stringify(state.lifecycle) === JSON.stringify(history)) return { state, lifecycle, recoveryRequired: state.recoveryRequired, problem: null };
|
||||
return { state: validState(state) ? state : null, lifecycle, recoveryRequired: true, problem: 'missing-or-corrupt-authoritative-state:retain-history-and-repair-manually' };
|
||||
}
|
||||
|
||||
export function writeFreshnessIssue(body: string, report: FreshnessReport): string {
|
||||
const region = [FRESHNESS_REGION_BEGIN, renderFreshnessReport(report), '', STATE_BEGIN, JSON.stringify(report.state), STATE_END, '', HISTORY_BEGIN, JSON.stringify(report.state.lifecycle), HISTORY_END, FRESHNESS_REGION_END].join('\n');
|
||||
let next: string;
|
||||
const owned = block(body, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END);
|
||||
if (malformedMarkers(body)) throw new Error('corrupt-owned-region:manual-recovery-required');
|
||||
if (owned !== null) {
|
||||
const recovery = readFreshnessIssue(body);
|
||||
const archive = recovery.problem ? `\n\n<details><summary>Corrupt authoritative region retained for manual recovery (base64 UTF-8)</summary>\n\n\`\`\`text\n${Buffer.from(owned.text, 'utf8').toString('base64')}\n\`\`\`\n</details>` : '';
|
||||
next = body.slice(0, owned.start) + region + archive + body.slice(owned.end);
|
||||
} else if (markerLines(body).some(line => line.text !== FRESHNESS_ISSUE_MARKER)) {
|
||||
throw new Error('corrupt-owned-region:manual-recovery-required');
|
||||
} else {
|
||||
next = `${body}${body ? '\n\n' : ''}${region}`;
|
||||
}
|
||||
const topMarkers = occurrences(body, FRESHNESS_ISSUE_MARKER);
|
||||
if (topMarkers > 1) throw new Error('duplicate-tracking-marker:manual-recovery-required');
|
||||
if (!topMarkers) {
|
||||
if (markerLines(body).length) throw new Error('corrupt-tracking-marker:manual-recovery-required');
|
||||
next = `${FRESHNESS_ISSUE_MARKER}\n${next}`;
|
||||
}
|
||||
if (new TextEncoder().encode(next).length > FRESHNESS_BOUNDS.issueBytes) throw new Error('tracking-issue-size-limit:manual-triage-required');
|
||||
return next;
|
||||
}
|
||||
|
||||
export type PublicationResult = { action: 'report-only' | 'rejected-obsolete-catalog' | 'rejected-older-observation' | 'updated' | 'created'; issueNumber?: number; report?: FreshnessReport };
|
||||
|
||||
async function checkAuthority(github: FreshnessGitHub, catalog: CatalogIdentity, signal: AbortSignal): Promise<boolean> {
|
||||
const current = await github.defaultCatalog(signal);
|
||||
if (signal.aborted) throw new Error('publication-aborted');
|
||||
if (current.defaultBranch !== 'main') throw new Error('default-branch-is-not-main:publication-disabled');
|
||||
return catalog.sourceSha256 === current.sourceSha256;
|
||||
}
|
||||
|
||||
function publicationReport(observation: FreshnessObservation, issue?: TrackingIssue): FreshnessReport | null {
|
||||
const stored = issue ? readFreshnessIssue(issue.body) : { state: null, lifecycle: [], recoveryRequired: false };
|
||||
if (stored.state && observationIsOlder(observation, stored.state.latest)) return null;
|
||||
if (stored.lifecycle.some(finding => Date.parse(finding.lastObservedAt) > Date.parse(observation.checkedAt))) return null;
|
||||
const report = assessFreshness(observation, stored.state, stored.lifecycle, stored.recoveryRequired);
|
||||
if (report.state.lastSuccess && !sameCatalog(observation.catalog, report.state.lastSuccess.catalog) && report.status === 'current') throw new Error('mismatched-evidence');
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function publishFreshness(observation: FreshnessObservation, github: FreshnessGitHub, ref: string): Promise<PublicationResult> {
|
||||
if (ref !== 'refs/heads/main') return { action: 'report-only', report: assessFreshness(observation) };
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
const publish = async (): Promise<PublicationResult> => {
|
||||
if (!await checkAuthority(github, observation.catalog, signal)) return { action: 'rejected-obsolete-catalog' };
|
||||
const issues = (await github.listIssues(signal)).filter(issue => markerLines(issue.body).length > 0);
|
||||
if (issues.some(issue => issue.author !== FRESHNESS_ISSUE_AUTHOR)) throw new Error('untrusted-tracking-marker-owner:manual-recovery-required');
|
||||
if (issues.length > 1 || issues.some(issue => occurrences(issue.body, FRESHNESS_ISSUE_MARKER) > 1)) throw new Error('duplicate-tracking-marker:manual-recovery-required');
|
||||
if (issues.some(issue => occurrences(issue.body, FRESHNESS_ISSUE_MARKER) !== 1 || malformedMarkers(issue.body))) throw new Error('corrupt-tracking-marker:manual-recovery-required');
|
||||
let issue = issues[0];
|
||||
if (!await checkAuthority(github, observation.catalog, signal)) return { action: 'rejected-obsolete-catalog' };
|
||||
if (issue) issue = await github.readIssue(issue.number, signal);
|
||||
if (issue && (occurrences(issue.body, FRESHNESS_ISSUE_MARKER) !== 1 || malformedMarkers(issue.body) || issue.author !== FRESHNESS_ISSUE_AUTHOR)) throw new Error('tracking-marker-or-owner-changed');
|
||||
const report = publicationReport(observation, issue);
|
||||
if (!report) return { action: 'rejected-older-observation', issueNumber: issue.number };
|
||||
const body = writeFreshnessIssue(issue?.body ?? '', report);
|
||||
const state = report.status === 'current' && report.complete && !report.state.lifecycle.some(finding => !finding.resolvedByCatalog) ? 'closed' : 'open';
|
||||
if (signal.aborted) throw new Error('publication-aborted');
|
||||
if (!issue) {
|
||||
issue = await github.createIssue(body, signal);
|
||||
if (issue.author !== FRESHNESS_ISSUE_AUTHOR) throw new Error('created-issue-owner-mismatch:manual-recovery-required');
|
||||
if (state === 'closed') await github.updateIssue(issue.number, { body, state }, signal);
|
||||
return { action: 'created', issueNumber: issue.number, report };
|
||||
}
|
||||
await github.updateIssue(issue.number, { body, state }, signal);
|
||||
return { action: 'updated', issueNumber: issue.number, report };
|
||||
};
|
||||
return withinDeadline(publish(), FRESHNESS_BOUNDS.publicationMs, controller);
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { marked, type Tokens } from 'marked';
|
||||
|
||||
export const FRESHNESS_SOURCES = [
|
||||
{ id: 'anthropic-models', url: 'https://platform.claude.com/docs/en/models/overview.md' },
|
||||
{ id: 'anthropic-lifecycle', url: 'https://platform.claude.com/docs/en/about-claude/model-deprecations.md' },
|
||||
{ id: 'openai-models', url: 'https://developers.openai.com/api/docs/models.md' },
|
||||
{ id: 'openai-lifecycle', url: 'https://developers.openai.com/api/docs/deprecations.md' },
|
||||
] as const;
|
||||
export type SourceId = typeof FRESHNESS_SOURCES[number]['id'];
|
||||
export const FRESHNESS_BOUNDS = { responseMs: 30_000, responseBytes: 2 * 1024 * 1024, redirects: 3, checkMs: 180_000, publicationMs: 60_000, issueBytes: 60_000 } as const;
|
||||
export type Fetcher = (url: string, init: RequestInit) => Promise<Response>;
|
||||
export type LifecycleRow = { modelId: string; state: 'active' | 'legacy' | 'deprecated' | 'removed'; deadline: string | null; supportFloor: string | null };
|
||||
export type ParsedSource = { recommendations?: { frontier: string; smart: string }; lineup?: string[]; lifecycle?: LifecycleRow[] };
|
||||
export type SourceReceipt = { id: SourceId; url: string; finalUrl: string; sha256: string; bytes: number; parsed: ParsedSource };
|
||||
export type SourceResult = { id: SourceId; receipt: SourceReceipt; error?: never } | { id: SourceId; error: string; receipt?: never };
|
||||
export const sha256 = (text: string | Uint8Array): string => createHash('sha256').update(text).digest('hex');
|
||||
|
||||
export async function withinDeadline<T>(task: Promise<T>, milliseconds: number, controller?: AbortController): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout>;
|
||||
const expiry = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => { controller?.abort(); reject(new Error('deadline-exceeded')); }, milliseconds);
|
||||
});
|
||||
try { return await Promise.race([task, expiry]); } finally { clearTimeout(timer!); }
|
||||
}
|
||||
|
||||
function allowedUrl(value: string): boolean {
|
||||
return FRESHNESS_SOURCES.some(source => source.url === value)
|
||||
|| value === 'https://developers.openai.com/api/docs/deprecations';
|
||||
}
|
||||
|
||||
export async function readBoundedResponse(response: Response): Promise<Uint8Array> {
|
||||
if (Number(response.headers.get('content-length')) > FRESHNESS_BOUNDS.responseBytes) {
|
||||
await response.body?.cancel(); throw new Error('response-too-large');
|
||||
}
|
||||
if (!response.body) throw new Error('empty-response');
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let bytes = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
bytes += chunk.value.length;
|
||||
if (bytes > FRESHNESS_BOUNDS.responseBytes) throw new Error('response-too-large');
|
||||
chunks.push(chunk.value);
|
||||
}
|
||||
} finally { await reader.cancel().catch(() => {}); }
|
||||
const data = new Uint8Array(bytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) { data.set(chunk, offset); offset += chunk.length; }
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function fetchOfficialSource(id: SourceId, fetcher: Fetcher = fetch, parentSignal?: AbortSignal): Promise<{ text: string; finalUrl: string; bytes: number }> {
|
||||
const source = FRESHNESS_SOURCES.find(source => source.id === id)!;
|
||||
const controller = new AbortController();
|
||||
const signal = parentSignal ? AbortSignal.any([parentSignal, controller.signal]) : controller.signal;
|
||||
const task = async () => {
|
||||
let url: string = source.url;
|
||||
for (let redirects = 0; ; redirects++) {
|
||||
if (!allowedUrl(url)) throw new Error('redirect-not-allowlisted');
|
||||
const response = await fetcher(url, { redirect: 'manual', signal, headers: { Accept: 'text/markdown, text/plain' } });
|
||||
if (response.status >= 300 && response.status <= 399) {
|
||||
await response.body?.cancel();
|
||||
if (redirects >= FRESHNESS_BOUNDS.redirects) throw new Error('redirect-limit');
|
||||
const location = response.headers.get('location');
|
||||
if (!location) throw new Error('redirect-without-location');
|
||||
url = new URL(location, url).href;
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) { await response.body?.cancel(); throw new Error(`http-${response.status}`); }
|
||||
const data = await readBoundedResponse(response);
|
||||
return { text: new TextDecoder('utf-8', { fatal: true }).decode(data), finalUrl: url, bytes: data.length };
|
||||
}
|
||||
};
|
||||
return withinDeadline(task(), FRESHNESS_BOUNDS.responseMs, controller);
|
||||
}
|
||||
|
||||
function section(text: string, title: string): string {
|
||||
const tokens = marked.lexer(text);
|
||||
if (tokens.filter(token => token.type === 'heading' && token.text === title).length !== 1) throw new Error(`missing-or-ambiguous-section:${title}`);
|
||||
const start = tokens.findIndex(token => token.type === 'heading' && token.text === title);
|
||||
if (start < 0) throw new Error(`missing-section:${title}`);
|
||||
const heading = tokens[start] as { depth: number };
|
||||
const end = tokens.findIndex((token, index) => index > start && token.type === 'heading' && token.depth <= heading.depth);
|
||||
return tokens.slice(start + 1, end < 0 ? undefined : end).map(token => token.raw).join('');
|
||||
}
|
||||
|
||||
function tables(text: string): string[][][] {
|
||||
return marked.lexer(text).flatMap(token => {
|
||||
if (token.type !== 'table') return [];
|
||||
const table = token as Tokens.Table;
|
||||
return [[table.header.map(cell => cell.text), ...table.rows.map(row => row.map(cell => cell.text))]];
|
||||
});
|
||||
}
|
||||
|
||||
function modelId(text: string): string {
|
||||
const id = text.replace(/^`|`$/g, '').trim();
|
||||
if (!/^[a-z][a-z0-9.-]{0,119}$/.test(id)) throw new Error('invalid-model-id');
|
||||
return id;
|
||||
}
|
||||
|
||||
export function lifecycleDate(value: string): string | null {
|
||||
if (/^(N\/A|To be announced|-)$/i.test(value.trim())) return null;
|
||||
const normalized = value.trim().replace(/\p{Dash_Punctuation}/gu, '-');
|
||||
if (/^\d{4}-\d\d-\d\d$/.test(normalized)) {
|
||||
const date = new Date(`${normalized}T00:00:00.000Z`);
|
||||
if (!Number.isFinite(date.valueOf()) || date.toISOString().slice(0, 10) !== normalized) throw new Error('malformed-lifecycle-date');
|
||||
return normalized;
|
||||
}
|
||||
const match = /^(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?) (\d{1,2}), (\d{4})$/.exec(value.trim());
|
||||
if (!match) throw new Error('malformed-lifecycle-date');
|
||||
const month = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'].indexOf(match[1].slice(0, 3));
|
||||
const date = new Date(Date.UTC(Number(match[3]), month, Number(match[2])));
|
||||
if (date.getUTCMonth() !== month || date.getUTCDate() !== Number(match[2])) throw new Error('malformed-lifecycle-date');
|
||||
return date.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function anthropicModels(text: string): ParsedSource {
|
||||
const compare = section(text, 'Compare models');
|
||||
const table = tables(compare).find(table => table[0][0] === 'Feature');
|
||||
if (!table || table[0].length < 3) throw new Error('missing-comparison-table');
|
||||
const ids = table.find(row => row[0] === 'Claude API ID')?.slice(1).map(modelId);
|
||||
const pages = table.find(row => row[0] === 'Model page')?.slice(1);
|
||||
if (!ids || !pages || ids.length !== pages.length) throw new Error('missing-api-id-row');
|
||||
if (new Set(ids).size !== ids.length || !pages.every(page => /\]\(https:\/\/platform\.claude\.com\/docs\/en\/models\/[a-z0-9-]+\/overview\)/.test(page))) throw new Error('ambiguous-comparison-models');
|
||||
const prose = marked.lexer(compare).filter(token => token.type === 'paragraph').map(token => token.raw).join('\n');
|
||||
const smartLinks = [...prose.matchAll(/start with \[[^\]]+\]\(([^)]+)\) for most workloads/gi)];
|
||||
const frontierLinks = [...prose.matchAll(/Use \[[^\]]+\]\(([^)]+)\) for demanding reasoning/gi)];
|
||||
if (smartLinks.length !== 1 || frontierLinks.length !== 1) throw new Error('unrecognized-recommendation-structure');
|
||||
const smart = smartLinks[0][1];
|
||||
const frontier = frontierLinks[0][1];
|
||||
const resolve = (url?: string): string => {
|
||||
const index = pages.findIndex(page => /\]\(([^)]+)\)/.exec(page)?.[1] === url);
|
||||
if (!url || index < 0) throw new Error('unrecognized-recommendation-structure');
|
||||
return ids[index];
|
||||
};
|
||||
return { recommendations: { frontier: resolve(frontier), smart: resolve(smart) }, lineup: ids };
|
||||
}
|
||||
|
||||
function openaiModels(text: string): ParsedSource {
|
||||
const featured = section(text, 'Featured models');
|
||||
const links = [...featured.matchAll(/^- \[[^\]]+\]\(\/api\/docs\/models\/([a-z0-9.-]+)\.md\): (.+)$/gm)];
|
||||
const choose = (pattern: RegExp): string => {
|
||||
const found = links.filter(link => pattern.test(link[2]));
|
||||
if (found.length !== 1) throw new Error('unrecognized-recommendation-structure');
|
||||
return modelId(found[0][1]);
|
||||
};
|
||||
const catalog = section(text, 'Browse our full catalog of models');
|
||||
const lineup = [...catalog.matchAll(/^- \[[^\]]+\]\(\/api\/docs\/models\/([a-z0-9.-]+)\.md\): /gm)].map(match => modelId(match[1]));
|
||||
if (!lineup.length) throw new Error('missing-model-catalog');
|
||||
const recommendations = { frontier: choose(/^Start here for complex reasoning and coding\.$/i), smart: choose(/^Balance intelligence and cost\.$/i) };
|
||||
if (!Object.values(recommendations).every(id => lineup.includes(id))) throw new Error('recommendation-not-in-catalog');
|
||||
return { recommendations, lineup };
|
||||
}
|
||||
|
||||
function anthropicLifecycle(text: string): ParsedSource {
|
||||
const table = tables(section(text, 'Model status')).find(table => table[0][0] === 'API model name');
|
||||
if (!table || table[0].join('|') !== 'API model name|Current state|Deprecated|Tentative retirement date' || table.length < 2) throw new Error('missing-lifecycle-table');
|
||||
const lifecycle = table.slice(1).map(row => {
|
||||
const states: Record<string, LifecycleRow['state']> = { Active: 'active', Legacy: 'legacy', Deprecated: 'deprecated', Retired: 'removed' };
|
||||
const state = states[row[1]];
|
||||
if (!state || row.length !== 4) throw new Error('unrecognized-lifecycle-state');
|
||||
lifecycleDate(row[2]);
|
||||
const floor = row[3].startsWith('Not sooner than ');
|
||||
const date = lifecycleDate(row[3].replace(/^Not sooner than /, ''));
|
||||
if (floor && (state === 'deprecated' || state === 'removed')) throw new Error('contradictory-lifecycle-floor');
|
||||
return { modelId: modelId(row[0]), state, deadline: floor ? null : date, supportFloor: floor ? date : null };
|
||||
});
|
||||
return { lifecycle };
|
||||
}
|
||||
|
||||
function openaiLifecycle(text: string): ParsedSource {
|
||||
const upcoming = section(text, 'Upcoming deprecations');
|
||||
const allTables = tables(text);
|
||||
const modelTables = allTables.filter(table => table[0][0] === 'Shutdown date' && /^(?:Model (?:\/ system|family \/ snapshot|snapshot)|Deprecated model|Legacy model)$/.test(table[0][1]));
|
||||
if (!upcoming || !modelTables.length) throw new Error('missing-deprecation-tables');
|
||||
const lifecycle = modelTables.flatMap(table => table.slice(1).flatMap(row => {
|
||||
if (/^(?:Videos API|Assistants API|OpenAI-Beta: |New fine-tuning training on )/.test(row[1]) || row[1].startsWith('`/')) return [];
|
||||
if (!row[1].startsWith('`')) throw new Error('unrecognized-deprecated-model-cell');
|
||||
const floor = row[0].startsWith('at earliest ');
|
||||
const date = lifecycleDate(row[0].replace(/^at earliest /, ''));
|
||||
if (!date) throw new Error('missing-shutdown-date');
|
||||
const ids = [...row[1].matchAll(/`([a-z0-9.-]+)`/g)].map(match => modelId(match[1]));
|
||||
if (!ids.length) throw new Error('unrecognized-deprecated-model-cell');
|
||||
return ids.map(modelId => ({ modelId, state: 'deprecated' as const, deadline: floor ? null : date, supportFloor: floor ? date : null }));
|
||||
}));
|
||||
return { lifecycle };
|
||||
}
|
||||
|
||||
export function parseOfficialSource(id: SourceId, text: string): ParsedSource {
|
||||
if (new TextEncoder().encode(text).length > FRESHNESS_BOUNDS.responseBytes) throw new Error('response-too-large');
|
||||
const parsers = { 'anthropic-models': anthropicModels, 'anthropic-lifecycle': anthropicLifecycle, 'openai-models': openaiModels, 'openai-lifecycle': openaiLifecycle };
|
||||
return parsers[id](text);
|
||||
}
|
||||
|
||||
export function parseOfficialSourceBounded(id: SourceId, text: string, signal: AbortSignal, createWorker: () => Worker = () => new Worker(new URL('./model-policy-freshness-parser-worker.ts', import.meta.url).href)): Promise<ParsedSource> {
|
||||
if (signal.aborted) return Promise.reject(new Error('deadline-exceeded'));
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = createWorker();
|
||||
const cleanup = () => { signal.removeEventListener('abort', abort); worker.terminate(); };
|
||||
const abort = () => { cleanup(); reject(new Error('deadline-exceeded')); };
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
worker.onmessage = (event: MessageEvent<{ ok: boolean; parsed: ParsedSource; error: string }>) => {
|
||||
cleanup();
|
||||
if (event.data.ok) resolve(event.data.parsed); else reject(new Error(event.data.error));
|
||||
};
|
||||
worker.onerror = () => { cleanup(); reject(new Error('parser-worker-unavailable')); };
|
||||
try { worker.postMessage({ id, text }); } catch { cleanup(); reject(new Error('parser-worker-unavailable')); }
|
||||
});
|
||||
}
|
||||
|
||||
export async function collectFreshnessSources(fetcher: Fetcher = fetch, createWorker?: () => Worker): Promise<SourceResult[]> {
|
||||
const controller = new AbortController();
|
||||
const completed: Partial<Record<SourceId, SourceResult>> = {};
|
||||
const tasks = Promise.all(FRESHNESS_SOURCES.map(async source => {
|
||||
let result: SourceResult;
|
||||
try {
|
||||
const fetched = await fetchOfficialSource(source.id, fetcher, controller.signal);
|
||||
const parsed = await parseOfficialSourceBounded(source.id, fetched.text, controller.signal, createWorker);
|
||||
result = { id: source.id, receipt: { id: source.id, url: source.url, finalUrl: fetched.finalUrl, sha256: sha256(fetched.text), bytes: fetched.bytes, parsed } };
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : 'source-unavailable';
|
||||
result = { id: source.id, error: /^[a-z0-9: /.-]{1,160}$/i.test(reason) ? reason : 'source-unavailable' };
|
||||
}
|
||||
completed[source.id] = result;
|
||||
return result;
|
||||
}));
|
||||
try { return await withinDeadline(tasks, FRESHNESS_BOUNDS.checkMs, controller); }
|
||||
catch { return FRESHNESS_SOURCES.map(source => completed[source.id] ?? { id: source.id, error: 'complete-check-deadline-exceeded' }); }
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { FRESHNESS_SOURCES, type SourceResult, type SourceReceipt } from './model-policy-freshness-sources';
|
||||
import { MODEL_CATALOG, MODEL_CATALOG_VERSION, modelCatalogSha256 } from './model-catalog';
|
||||
import { sha256 } from './model-policy-freshness-sources';
|
||||
|
||||
export const FRESHNESS_SCHEMA_VERSION = 1;
|
||||
export const FRESHNESS_PARSER_VERSION = 1;
|
||||
export type FreshnessModel = { provider: 'anthropic' | 'openai'; tier: 'frontier' | 'smart'; modelId: string };
|
||||
export type CatalogIdentity = { sha256: string; sourceSha256: string; version: number; models: FreshnessModel[] };
|
||||
export type FreshnessRun = { id: string; attempt: number; startedAt: string; headSha: string };
|
||||
export type FreshnessObservation = { catalog: CatalogIdentity; checkedAt: string; run: FreshnessRun; sources: SourceResult[] };
|
||||
export type FreshnessStatus = 'current' | 'update-candidate' | 'stale' | 'unknown/source-unavailable';
|
||||
export type LifecycleFinding = { key: string; provider: FreshnessModel['provider']; modelId: string; state: 'legacy' | 'deprecated' | 'removed'; deadline: string | null; firstObservedAt: string; lastObservedAt: string; sourceUrl: string; resolvedByCatalog?: string };
|
||||
export type FreshnessEvidence = {
|
||||
schemaVersion: 1; parserVersion: number; catalog: CatalogIdentity; checkedAt: string; nextCheckDueBy: string; run: FreshnessRun;
|
||||
sources: Omit<SourceReceipt, 'parsed'>[];
|
||||
};
|
||||
export type FreshnessState = {
|
||||
schemaVersion: 1; latest: { checkedAt: string; run: FreshnessRun }; lastSuccess: FreshnessEvidence | null;
|
||||
lifecycle: LifecycleFinding[]; recoveryRequired: boolean;
|
||||
};
|
||||
export type FreshnessReport = {
|
||||
schemaVersion: 1; status: FreshnessStatus; evidenceStatus: FreshnessStatus; complete: boolean;
|
||||
errors: string[]; candidates: { provider: string; tier: string; current: string; proposed: string; reason: string }[];
|
||||
state: FreshnessState;
|
||||
};
|
||||
const DAY = 86_400_000;
|
||||
const iso = (value: string): boolean => /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(value) && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value;
|
||||
const hash = (value: unknown): value is string => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value);
|
||||
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === 'object' && !Array.isArray(value);
|
||||
const string = (value: unknown): value is string => typeof value === 'string';
|
||||
export const isFreshnessDate = (value: unknown): value is string => string(value) && iso(value);
|
||||
|
||||
export function freshnessCatalogIdentity(sourceText: string): CatalogIdentity {
|
||||
return { sha256: modelCatalogSha256(), sourceSha256: sha256(sourceText), version: MODEL_CATALOG_VERSION, models: MODEL_CATALOG.map(entry => ({ provider: entry.provider, tier: entry.tier, modelId: entry.model })) };
|
||||
}
|
||||
|
||||
export function validCatalog(value: unknown): value is CatalogIdentity {
|
||||
if (!object(value) || !hash(value.sha256) || !hash(value.sourceSha256) || !Number.isInteger(value.version) || Number(value.version) < 1 || !Array.isArray(value.models) || value.models.length !== 4) return false;
|
||||
const keys = new Set<string>();
|
||||
for (const entry of value.models) {
|
||||
if (!object(entry) || !['anthropic', 'openai'].includes(String(entry.provider)) || !['frontier', 'smart'].includes(String(entry.tier)) || !string(entry.modelId) || !/^[a-z][a-z0-9.-]{0,119}$/.test(entry.modelId)) return false;
|
||||
keys.add(`${entry.provider}/${entry.tier}`);
|
||||
}
|
||||
return keys.size === 4;
|
||||
}
|
||||
|
||||
export function validRun(value: unknown): value is FreshnessRun {
|
||||
return object(value) && string(value.id) && /^\d{1,24}$/.test(value.id) && Number.isSafeInteger(value.attempt) && Number(value.attempt) > 0 && isFreshnessDate(value.startedAt) && string(value.headSha) && /^[a-f0-9]{40,64}$/.test(value.headSha);
|
||||
}
|
||||
|
||||
function validReceipt(value: unknown, source: typeof FRESHNESS_SOURCES[number]): boolean {
|
||||
return object(value) && value.id === source.id && value.url === source.url && hash(value.sha256)
|
||||
&& (FRESHNESS_SOURCES.some(item => item.url === value.finalUrl) || value.finalUrl === 'https://developers.openai.com/api/docs/deprecations')
|
||||
&& Number.isInteger(value.bytes) && Number(value.bytes) > 0 && Number(value.bytes) <= 2 * 1024 * 1024;
|
||||
}
|
||||
|
||||
export function sameCatalog(a: CatalogIdentity, b: CatalogIdentity): boolean {
|
||||
const keys = (catalog: CatalogIdentity) => catalog.models.map(model => `${model.provider}/${model.tier}/${model.modelId}`).sort().join('|');
|
||||
return a.sha256 === b.sha256 && a.sourceSha256 === b.sourceSha256 && a.version === b.version && keys(a) === keys(b);
|
||||
}
|
||||
|
||||
export function validLifecycle(value: unknown): value is LifecycleFinding[] {
|
||||
return Array.isArray(value) && value.length <= 128 && new Set(value.map(item => object(item) ? item.key : undefined)).size === value.length && value.every(item => object(item)
|
||||
&& ['anthropic', 'openai'].includes(String(item.provider)) && string(item.modelId) && /^[a-z][a-z0-9.-]{0,119}$/.test(item.modelId)
|
||||
&& item.key === `${item.provider}/${item.modelId}` && ['legacy', 'deprecated', 'removed'].includes(String(item.state))
|
||||
&& (item.deadline === null || (string(item.deadline) && /^\d{4}-\d\d-\d\d$/.test(item.deadline) && isFreshnessDate(`${item.deadline}T00:00:00.000Z`)))
|
||||
&& isFreshnessDate(item.firstObservedAt) && isFreshnessDate(item.lastObservedAt)
|
||||
&& FRESHNESS_SOURCES.some(source => source.id === `${item.provider}-lifecycle` && source.url === item.sourceUrl)
|
||||
&& (item.resolvedByCatalog === undefined || hash(item.resolvedByCatalog)));
|
||||
}
|
||||
|
||||
export function validEvidence(value: unknown): value is FreshnessEvidence {
|
||||
if (!object(value) || value.schemaVersion !== 1 || value.parserVersion !== FRESHNESS_PARSER_VERSION || !validCatalog(value.catalog) || !isFreshnessDate(value.checkedAt) || !isFreshnessDate(value.nextCheckDueBy) || !validRun(value.run) || !Array.isArray(value.sources) || value.sources.length !== 4) return false;
|
||||
if (Date.parse(value.nextCheckDueBy) !== Date.parse(value.checkedAt) + 7 * DAY) return false;
|
||||
const sources = value.sources;
|
||||
return Date.parse(value.run.startedAt) <= Date.parse(value.checkedAt)
|
||||
&& FRESHNESS_SOURCES.every(source => sources.filter(receipt => validReceipt(receipt, source)).length === 1);
|
||||
}
|
||||
|
||||
export function validState(value: unknown): value is FreshnessState {
|
||||
if (!object(value) || value.schemaVersion !== 1 || !object(value.latest) || !isFreshnessDate(value.latest.checkedAt) || !validRun(value.latest.run)
|
||||
|| (value.lastSuccess !== null && !validEvidence(value.lastSuccess)) || !validLifecycle(value.lifecycle) || typeof value.recoveryRequired !== 'boolean') return false;
|
||||
if (Date.parse(value.latest.run.startedAt) > Date.parse(value.latest.checkedAt)) return false;
|
||||
if (value.lastSuccess && (Date.parse(value.lastSuccess.checkedAt) > Date.parse(value.latest.checkedAt) || BigInt(value.lastSuccess.run.id) > BigInt(value.latest.run.id))) return false;
|
||||
const checkedAt = value.latest.checkedAt;
|
||||
return value.lifecycle.every(finding => Date.parse(finding.firstObservedAt) <= Date.parse(finding.lastObservedAt) && Date.parse(finding.lastObservedAt) <= Date.parse(checkedAt));
|
||||
}
|
||||
|
||||
export function storedEvidenceStatus(catalog: CatalogIdentity, evidence: FreshnessEvidence | null, now: string): FreshnessStatus {
|
||||
if (!evidence || !validEvidence(evidence) || !sameCatalog(catalog, evidence.catalog) || !isFreshnessDate(now) || Date.parse(evidence.checkedAt) > Date.parse(now)) return 'unknown/source-unavailable';
|
||||
return Date.parse(now) - Date.parse(evidence.checkedAt) > 30 * DAY ? 'stale' : 'current';
|
||||
}
|
||||
|
||||
export function observationIsOlder(observation: FreshnessObservation, latest: FreshnessState['latest']): boolean {
|
||||
const a = observation.run;
|
||||
const b = latest.run;
|
||||
return BigInt(a.id) < BigInt(b.id) || (a.id === b.id && a.attempt <= b.attempt)
|
||||
|| Date.parse(observation.checkedAt) < Date.parse(latest.checkedAt)
|
||||
|| Date.parse(a.startedAt) < Date.parse(b.startedAt);
|
||||
}
|
||||
|
||||
function sourceErrors(observation: FreshnessObservation): string[] {
|
||||
return FRESHNESS_SOURCES.flatMap(source => {
|
||||
const results = observation.sources.filter(result => result.id === source.id);
|
||||
if (results.length !== 1) return [`${source.id}:missing-or-duplicate-source`];
|
||||
const result = results[0];
|
||||
if (result.error !== undefined) return [`${source.id}:${result.error}`];
|
||||
const receipt = result.receipt;
|
||||
if (!validReceipt(receipt, source)) return [`${source.id}:invalid-receipt`];
|
||||
if (source.id.endsWith('-models') && (!receipt.parsed.recommendations || !receipt.parsed.lineup?.length)) return [`${source.id}:missing-recommendations`];
|
||||
if (source.id.endsWith('-lifecycle') && !receipt.parsed.lifecycle?.length) return [`${source.id}:missing-lifecycle`];
|
||||
return [];
|
||||
});
|
||||
}
|
||||
|
||||
function family(id: string): string {
|
||||
if (id.startsWith('claude-')) return id.split('-')[1];
|
||||
return /^gpt-\d+(?:\.\d+)?-([a-z]+)/.exec(id)?.[1] ?? id;
|
||||
}
|
||||
|
||||
export function assessFreshness(observation: FreshnessObservation, previous: FreshnessState | null = null, recoveredLifecycle: LifecycleFinding[] = [], recoveryRequired = false): FreshnessReport {
|
||||
if (!validCatalog(observation.catalog) || !validRun(observation.run) || !isFreshnessDate(observation.checkedAt) || Date.parse(observation.checkedAt) < Date.parse(observation.run.startedAt)) throw new Error('invalid-observation-identity');
|
||||
const lifecycle = new Map((previous?.lifecycle ?? recoveredLifecycle).map(finding => [finding.key, { ...finding }]));
|
||||
for (const finding of lifecycle.values()) {
|
||||
if (observation.catalog.models.some(model => model.provider === finding.provider && model.modelId === finding.modelId)) delete finding.resolvedByCatalog;
|
||||
}
|
||||
const errors = sourceErrors(observation);
|
||||
const candidates: FreshnessReport['candidates'] = [];
|
||||
for (const model of observation.catalog.models) {
|
||||
const models = observation.sources.find(source => source.id === `${model.provider}-models`)?.receipt?.parsed;
|
||||
const rows = observation.sources.find(source => source.id === `${model.provider}-lifecycle`)?.receipt?.parsed.lifecycle;
|
||||
if (model.provider === 'anthropic' && rows && !rows.some(row => row.modelId === model.modelId)) errors.push(`anthropic-lifecycle:missing-default:${model.modelId}`);
|
||||
const row = rows?.find(row => row.modelId === model.modelId);
|
||||
if (row && row.state !== 'active') {
|
||||
const key = `${model.provider}/${model.modelId}`;
|
||||
const previousFinding = lifecycle.get(key);
|
||||
const rank = { legacy: 1, deprecated: 2, removed: 3 };
|
||||
const observedState = row.state === 'deprecated' && row.deadline && row.deadline <= observation.checkedAt.slice(0, 10) ? 'removed' : row.state;
|
||||
const state = previousFinding && rank[previousFinding.state] > rank[observedState] ? previousFinding.state : observedState;
|
||||
const deadline = previousFinding?.deadline && (!row.deadline || previousFinding.deadline < row.deadline) ? previousFinding.deadline : row.deadline;
|
||||
lifecycle.set(key, { key, provider: model.provider, modelId: model.modelId, state, deadline, sourceUrl: FRESHNESS_SOURCES.find(source => source.id === `${model.provider}-lifecycle`)!.url, firstObservedAt: previousFinding?.firstObservedAt ?? observation.checkedAt, lastObservedAt: observation.checkedAt });
|
||||
}
|
||||
const proposed = models?.recommendations?.[model.tier];
|
||||
if (proposed && proposed !== model.modelId) candidates.push({ provider: model.provider, tier: model.tier, current: model.modelId, proposed, reason: family(proposed) === family(model.modelId) ? 'changed-monitored-family-recommendation' : 'unknown-replacement-family-needs-triage' });
|
||||
if (models?.lineup && !models.lineup.includes(model.modelId) && proposed === model.modelId) errors.push(`${model.provider}-models:inconsistent-default:${model.modelId}`);
|
||||
}
|
||||
const complete = errors.length === 0;
|
||||
if (complete) for (const finding of lifecycle.values()) {
|
||||
if (!observation.catalog.models.some(model => model.provider === finding.provider && model.modelId === finding.modelId)) finding.resolvedByCatalog = observation.catalog.sha256;
|
||||
}
|
||||
const recovery = recoveryRequired || previous?.recoveryRequired === true;
|
||||
if (recovery) errors.push('authoritative-state-recovery-required:manual-triage');
|
||||
const lastSuccess: FreshnessEvidence | null = complete && !recovery ? {
|
||||
schemaVersion: 1, parserVersion: FRESHNESS_PARSER_VERSION, catalog: observation.catalog, checkedAt: observation.checkedAt,
|
||||
nextCheckDueBy: new Date(Date.parse(observation.checkedAt) + 7 * DAY).toISOString(), run: observation.run,
|
||||
sources: observation.sources.map(source => { const { parsed, ...receipt } = source.receipt!; return receipt; }),
|
||||
} : previous?.lastSuccess ?? null;
|
||||
const evidenceStatus = storedEvidenceStatus(observation.catalog, lastSuccess, observation.checkedAt);
|
||||
const unresolved = [...lifecycle.values()].some(finding => !finding.resolvedByCatalog);
|
||||
const status: FreshnessStatus = errors.length ? 'unknown/source-unavailable' : unresolved || candidates.length ? 'update-candidate' : evidenceStatus;
|
||||
return { schemaVersion: 1, status, evidenceStatus, complete: complete && !recovery, errors, candidates, state: { schemaVersion: 1, latest: { checkedAt: observation.checkedAt, run: observation.run }, lastSuccess, lifecycle: [...lifecycle.values()].sort((a, b) => a.key.localeCompare(b.key)), recoveryRequired: recovery } };
|
||||
}
|
||||
|
||||
export function renderFreshnessReport(report: FreshnessReport): string {
|
||||
const evidence = report.state.lastSuccess;
|
||||
const lines = [
|
||||
`## Model-policy freshness: ${report.status}`, '',
|
||||
`Checked observation: ${report.state.latest.checkedAt}; run ${report.state.latest.run.id}/${report.state.latest.run.attempt}.`,
|
||||
`Last complete trustworthy check: ${evidence?.checkedAt ?? 'unavailable'}. Next scheduled check due by: ${evidence?.nextCheckDueBy ?? 'unavailable'}.`,
|
||||
`Stored evidence status: ${report.evidenceStatus}; trustworthy evidence older than thirty days is stale, independently of the seven-day scheduled due date.`,
|
||||
'The seven-day due date is a manual-check mitigation, not automatic liveness monitoring. If GitHub never runs this job, it cannot alert on its own absence.',
|
||||
'Public API recommendations only; this does not certify account entitlement, quality superiority, CLI compatibility, or Bedrock/Vertex/custom endpoint availability.', '',
|
||||
];
|
||||
for (const error of report.errors) lines.push(`- Source/recovery finding: ${error}. Triage within seven days; manually check the named official source and the last successful workflow date.`);
|
||||
for (const candidate of report.candidates) lines.push(`- Candidate (${candidate.provider}/${candidate.tier}): ${candidate.current} → ${candidate.proposed}; ${candidate.reason}. Review within thirty days; qualify a reviewed catalog PR or record a retain decision.`);
|
||||
for (const finding of report.state.lifecycle.filter(finding => !finding.resolvedByCatalog)) lines.push(`- ${finding.state === 'legacy' ? 'Lifecycle triage' : 'URGENT'}: ${finding.modelId} is ${finding.state}; removal deadline ${finding.deadline ?? 'not announced'}. Source: ${finding.sourceUrl}. ${finding.state === 'legacy' ? 'Review within thirty days; legacy does not announce retirement.' : 'Triage within one working day and qualify or explicitly mitigate before removal; already-removed defaults remain unresolved.'}`);
|
||||
lines.push('', 'No catalog updates, pin rewrites, model switches, or paid evaluations are performed. Candidate qualification requires separate budget approval and pinned baseline/candidate inputs.', 'Manual recovery: docs/troubleshooting.md#model-policy-freshness. Preserve lifecycle history and dispositions when repairing the owned state; unresolved lifecycle findings cannot be cleared by a failed fetch.', '', '### Official source receipts');
|
||||
for (const receipt of evidence?.sources ?? []) lines.push(`- ${receipt.id}: ${receipt.url}; SHA-256 ${receipt.sha256}; ${receipt.bytes} bytes.`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/** One-time plan-review policy notice, emitted only at an actual role-bearing invocation. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { MODEL_POLICY_VERSION, PROVIDER_ENV_OVERRIDES, describeSelection, modelPolicyCommands, type ModelSelection } from './model-policy';
|
||||
import { resolveStateRoot, type StateRootEnv } from './state-root';
|
||||
|
||||
export function modelPolicyNoticeMarker(env: StateRootEnv = process.env, platform: NodeJS.Platform = process.platform): string {
|
||||
return path.join(resolveStateRoot(env, platform), `.model-policy-notice-v${MODEL_POLICY_VERSION}`);
|
||||
}
|
||||
|
||||
export function modelPolicyNoticeText(selection: ModelSelection): string {
|
||||
const commands = modelPolicyCommands();
|
||||
return [
|
||||
'NOTICE: gstack plan reviews now use an independent plan-review model (default: frontier tier), not your general coding-model setting.',
|
||||
`This review: ${describeSelection(selection)}.`,
|
||||
`Explicit choices still win, in order: a model named for the request, ${PROVIDER_ENV_OVERRIDES[selection.provider]}, then the per-tier config override.`,
|
||||
`Use the smart tier: ${commands.useSmart}. Keep host model settings: ${commands.useHost}. Shown once. Docs: https://github.com/garrytan/gstack/blob/main/docs/model-policy.md`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Show before the paid call; record the marker only afterwards. A marker that
|
||||
* cannot be written repeats the notice next time instead of hiding it.
|
||||
*/
|
||||
export function emitModelPolicyNotice(selection: ModelSelection, opts: {
|
||||
env?: StateRootEnv;
|
||||
platform?: NodeJS.Platform;
|
||||
write?: (text: string) => void;
|
||||
} = {}): boolean {
|
||||
let marker: string;
|
||||
try {
|
||||
marker = modelPolicyNoticeMarker(opts.env, opts.platform);
|
||||
if (fs.existsSync(marker)) return false;
|
||||
} catch {
|
||||
marker = '';
|
||||
}
|
||||
(opts.write ?? ((text: string) => process.stderr.write(text)))(`${modelPolicyNoticeText(selection)}\n`);
|
||||
if (!marker) return true;
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(marker), { recursive: true });
|
||||
fs.writeFileSync(marker, '');
|
||||
} catch {}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,661 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { catalogEntry, MODEL_CATALOG_VERSION, MODEL_PROVIDERS, type ModelCatalogEntry, type ModelProvider, type ModelTier } from './model-catalog';
|
||||
import { resolveStateRoot, type StateRootEnv } from './state-root';
|
||||
|
||||
export const MODEL_POLICY_VERSION = 1 as const;
|
||||
|
||||
export type ModelRole = 'plan-review' | 'implementation';
|
||||
export type PlanReviewTier = ModelTier | 'host';
|
||||
export type ModelHost = 'claude-code' | 'codex' | 'unknown';
|
||||
|
||||
export const MODEL_ROLES: readonly ModelRole[] = Object.freeze(['plan-review', 'implementation'] as const);
|
||||
export const PLAN_REVIEW_TIERS: readonly PlanReviewTier[] = Object.freeze(['frontier', 'smart', 'host'] as const);
|
||||
export const IMPLEMENTATION_TIERS: readonly ModelTier[] = Object.freeze(['frontier', 'smart'] as const);
|
||||
|
||||
export type PolicyConfigKey =
|
||||
| 'plan_review_tier'
|
||||
| 'implementation_tier'
|
||||
| 'model_frontier_claude'
|
||||
| 'model_frontier_openai'
|
||||
| 'model_smart_claude'
|
||||
| 'model_smart_openai';
|
||||
|
||||
export const POLICY_CONFIG_KEYS: readonly PolicyConfigKey[] = Object.freeze([
|
||||
'plan_review_tier',
|
||||
'implementation_tier',
|
||||
'model_frontier_claude',
|
||||
'model_frontier_openai',
|
||||
'model_smart_claude',
|
||||
'model_smart_openai',
|
||||
] as const);
|
||||
|
||||
export const MODEL_OVERRIDE_KEYS: Readonly<Record<ModelProvider, Readonly<Record<ModelTier, PolicyConfigKey>>>> = Object.freeze({
|
||||
anthropic: Object.freeze({ frontier: 'model_frontier_claude', smart: 'model_smart_claude' } as const),
|
||||
openai: Object.freeze({ frontier: 'model_frontier_openai', smart: 'model_smart_openai' } as const),
|
||||
});
|
||||
|
||||
export const PROVIDER_ENV_OVERRIDES = Object.freeze({ anthropic: 'GSTACK_CLAUDE_MODEL', openai: 'GSTACK_CODEX_MODEL' } as const);
|
||||
|
||||
export const OPENAI_MODEL_ID_PATTERN = /^[A-Za-z0-9._:/-]{1,100}$/;
|
||||
export const ANTHROPIC_MODEL_ID_PATTERN = /^[A-Za-z0-9._:/@+[\]-]{1,512}$/;
|
||||
|
||||
const DOCS = 'https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md';
|
||||
const DOCS_CONFIG = `${DOCS}#model-policy-config`;
|
||||
const DOCS_PROVIDER = `${DOCS}#model-policy-provider`;
|
||||
const DOCS_SELECTION = `${DOCS}#model-policy-selection`;
|
||||
|
||||
const PROVIDER_LABEL: Record<ModelProvider, string> = { anthropic: 'Anthropic (Claude Code)', openai: 'OpenAI (Codex)' };
|
||||
|
||||
export type ModelPolicyReason =
|
||||
| 'config_unreadable'
|
||||
| 'config_malformed'
|
||||
| 'empty_value'
|
||||
| 'invalid_plan_review_tier'
|
||||
| 'invalid_implementation_tier'
|
||||
| 'invalid_model_id'
|
||||
| 'custom_provider_requires_model'
|
||||
| 'native_provider_unresolved'
|
||||
| 'provider_required';
|
||||
|
||||
export type SelectionSource =
|
||||
| { kind: 'request'; label: string }
|
||||
| { kind: 'env'; name: (typeof PROVIDER_ENV_OVERRIDES)[ModelProvider]; label: string }
|
||||
| { kind: 'config'; key: PolicyConfigKey; path: string; label: string }
|
||||
| { kind: 'catalog'; catalogVersion: typeof MODEL_CATALOG_VERSION; verifiedAt: string; sourceUrl: string; label: string }
|
||||
| { kind: 'host'; reason: 'host-mode'; key: 'plan_review_tier'; label: string };
|
||||
|
||||
export interface ModelPolicyErrorJson {
|
||||
reason: ModelPolicyReason;
|
||||
problem: string;
|
||||
cause: string;
|
||||
key: string | null;
|
||||
source: SelectionSource['kind'] | 'native' | null;
|
||||
repair: string[];
|
||||
docs: string;
|
||||
}
|
||||
|
||||
export class ModelPolicyError extends Error {
|
||||
readonly reason: ModelPolicyReason;
|
||||
readonly problem: string;
|
||||
readonly cause: string;
|
||||
readonly key: string | null;
|
||||
readonly source: ModelPolicyErrorJson['source'];
|
||||
readonly repair: string[];
|
||||
readonly docs: string;
|
||||
|
||||
constructor(detail: ModelPolicyErrorJson) {
|
||||
super(`${detail.problem}: ${detail.cause}`);
|
||||
this.name = 'ModelPolicyError';
|
||||
this.reason = detail.reason;
|
||||
this.problem = detail.problem;
|
||||
this.cause = detail.cause;
|
||||
this.key = detail.key;
|
||||
this.source = detail.source;
|
||||
this.repair = detail.repair;
|
||||
this.docs = detail.docs;
|
||||
}
|
||||
|
||||
toJSON(): ModelPolicyErrorJson {
|
||||
return { reason: this.reason, problem: this.problem, cause: this.cause, key: this.key, source: this.source, repair: this.repair, docs: this.docs };
|
||||
}
|
||||
}
|
||||
|
||||
export interface TierChoice<T extends PlanReviewTier> {
|
||||
value: T;
|
||||
key: 'plan_review_tier' | 'implementation_tier';
|
||||
origin: 'config' | 'default';
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface ModelPolicyConfig {
|
||||
path: string;
|
||||
exists: boolean;
|
||||
planReviewTier: TierChoice<PlanReviewTier>;
|
||||
implementationTier: TierChoice<ModelTier>;
|
||||
overrides: Record<ModelProvider, Record<ModelTier, string | null>>;
|
||||
}
|
||||
|
||||
interface SelectionBase {
|
||||
role: ModelRole;
|
||||
provider: ModelProvider;
|
||||
tier: PlanReviewTier;
|
||||
tierFrom: TierChoice<PlanReviewTier>;
|
||||
catalog: ModelCatalogEntry | null;
|
||||
}
|
||||
|
||||
export interface SelectedModel extends SelectionBase {
|
||||
status: 'selected';
|
||||
requestedModel: string;
|
||||
source: Exclude<SelectionSource, { kind: 'host' }>;
|
||||
}
|
||||
|
||||
export interface DelegatedHostModel extends SelectionBase {
|
||||
status: 'delegated-host';
|
||||
requestedModel: null;
|
||||
source: Extract<SelectionSource, { kind: 'host' }>;
|
||||
}
|
||||
|
||||
export type ModelSelection = SelectedModel | DelegatedHostModel;
|
||||
|
||||
function sanitize(value: string): string {
|
||||
return value.replace(/[\x00-\x1f\x7f]/g, ' ').slice(0, 200);
|
||||
}
|
||||
|
||||
function providerOfKey(key: PolicyConfigKey): ModelProvider | null {
|
||||
for (const provider of MODEL_PROVIDERS) {
|
||||
if (Object.values(MODEL_OVERRIDE_KEYS[provider]).includes(key)) return provider;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isValidModelId(provider: ModelProvider, value: string): boolean {
|
||||
return (provider === 'openai' ? OPENAI_MODEL_ID_PATTERN : ANTHROPIC_MODEL_ID_PATTERN).test(value);
|
||||
}
|
||||
|
||||
function modelIdRule(provider: ModelProvider): string {
|
||||
return provider === 'openai'
|
||||
? '1-100 characters from A-Z a-z 0-9 . _ : / -'
|
||||
: '1-512 characters from A-Z a-z 0-9 . _ : / @ + [ ] -';
|
||||
}
|
||||
|
||||
export function validatePolicyValue(key: PolicyConfigKey, rawValue: string, configPath = 'config.yaml'): ModelPolicyError | null {
|
||||
const value = rawValue.trim();
|
||||
if (value === '') {
|
||||
return new ModelPolicyError({
|
||||
reason: 'empty_value',
|
||||
problem: `${key} is empty in ${configPath}`,
|
||||
cause: 'an empty policy value is not a choice; unset the key to use the default',
|
||||
key,
|
||||
source: 'config',
|
||||
repair: [`gstack-config unset ${key}`],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
if (key === 'plan_review_tier' && !(PLAN_REVIEW_TIERS as readonly string[]).includes(value)) {
|
||||
return new ModelPolicyError({
|
||||
reason: 'invalid_plan_review_tier',
|
||||
problem: `plan_review_tier '${sanitize(value)}' in ${configPath} is not recognized`,
|
||||
cause: 'valid values are frontier, smart and host',
|
||||
key,
|
||||
source: 'config',
|
||||
repair: ['gstack-config unset plan_review_tier'],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
if (key === 'implementation_tier' && !(IMPLEMENTATION_TIERS as readonly string[]).includes(value)) {
|
||||
return new ModelPolicyError({
|
||||
reason: 'invalid_implementation_tier',
|
||||
problem: `implementation_tier '${sanitize(value)}' in ${configPath} is not recognized`,
|
||||
cause: 'valid values are frontier and smart',
|
||||
key,
|
||||
source: 'config',
|
||||
repair: ['gstack-config unset implementation_tier'],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
const provider = providerOfKey(key);
|
||||
if (provider && !isValidModelId(provider, value)) {
|
||||
return new ModelPolicyError({
|
||||
reason: 'invalid_model_id',
|
||||
problem: `${key} '${sanitize(value)}' in ${configPath} is not a valid ${provider} model ID`,
|
||||
cause: `use ${modelIdRule(provider)}`,
|
||||
key,
|
||||
source: 'config',
|
||||
repair: [`gstack-config set ${key} <model-id>`, `gstack-config unset ${key}`],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseModelPolicyConfig(text: string, configPath: string): ModelPolicyConfig {
|
||||
const records = new Map<PolicyConfigKey, string>();
|
||||
for (const line of text.split('\n')) {
|
||||
for (const key of POLICY_CONFIG_KEYS) {
|
||||
if (line.startsWith(`${key}:`)) {
|
||||
records.set(key, line.slice(key.length + 1).trim());
|
||||
} else if (new RegExp(`^\\s*${key}\\s*:`).test(line)) {
|
||||
throw new ModelPolicyError({
|
||||
reason: 'config_malformed',
|
||||
problem: `${configPath} has a malformed ${key} record`,
|
||||
cause: `write it as '${key}: <value>' at the start of the line, with no indentation or space before the colon`,
|
||||
key,
|
||||
source: 'config',
|
||||
repair: [`gstack-config unset ${key}`, `gstack-config set ${key} <value>`],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const configValue = (key: PolicyConfigKey): string | null => {
|
||||
const value = records.get(key);
|
||||
if (value === undefined) return null;
|
||||
const problem = validatePolicyValue(key, value, configPath);
|
||||
if (problem) throw problem;
|
||||
return value;
|
||||
};
|
||||
const planReviewTier = configValue('plan_review_tier') as PlanReviewTier | null;
|
||||
const implementationTier = configValue('implementation_tier') as ModelTier | null;
|
||||
return {
|
||||
path: configPath,
|
||||
exists: true,
|
||||
planReviewTier: { value: planReviewTier ?? 'frontier', key: 'plan_review_tier', origin: planReviewTier ? 'config' : 'default', path: configPath },
|
||||
implementationTier: { value: implementationTier ?? 'smart', key: 'implementation_tier', origin: implementationTier ? 'config' : 'default', path: configPath },
|
||||
overrides: {
|
||||
anthropic: { frontier: configValue('model_frontier_claude'), smart: configValue('model_smart_claude') },
|
||||
openai: { frontier: configValue('model_frontier_openai'), smart: configValue('model_smart_openai') },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function readModelPolicyConfig(opts: { env?: StateRootEnv; platform?: NodeJS.Platform } = {}): ModelPolicyConfig {
|
||||
const configPath = `${resolveStateRoot(opts.env ?? process.env, opts.platform ?? process.platform)}/config.yaml`;
|
||||
let text: string;
|
||||
try {
|
||||
text = fs.readFileSync(configPath, 'utf-8');
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code ?? 'unknown error';
|
||||
if (code === 'ENOENT') return { ...parseModelPolicyConfig('', configPath), exists: false };
|
||||
const cause = code === 'EISDIR' ? 'config.yaml is a directory'
|
||||
: code === 'EACCES' || code === 'EPERM' ? 'permission denied'
|
||||
: code === 'ENOTDIR' ? 'the state root is not a directory'
|
||||
: `the read failed (${code})`;
|
||||
throw new ModelPolicyError({
|
||||
reason: 'config_unreadable',
|
||||
problem: `gstack cannot read ${configPath}, so the model policy is unknown and no outside model call will start`,
|
||||
cause,
|
||||
key: null,
|
||||
source: 'config',
|
||||
repair: [`fix the permissions or type of ${configPath}`, 'or point GSTACK_HOME at a readable state root'],
|
||||
docs: DOCS_CONFIG,
|
||||
});
|
||||
}
|
||||
return parseModelPolicyConfig(text, configPath);
|
||||
}
|
||||
|
||||
const TRUTHY = new Set(['1', 'true', 'yes', 'on']);
|
||||
|
||||
function truthy(value: unknown): boolean {
|
||||
return typeof value === 'string' && TRUTHY.has(value.trim().toLowerCase());
|
||||
}
|
||||
|
||||
function nonDefaultUrl(value: unknown, defaultUrl: string): boolean {
|
||||
return typeof value === 'string' && value.trim() !== '' && value.trim().replace(/\/+$/, '') !== defaultUrl;
|
||||
}
|
||||
|
||||
function homeDir(env: StateRootEnv, platform: NodeJS.Platform): string {
|
||||
return env.HOME || (platform === 'win32' ? env.USERPROFILE || '' : '') || os.homedir();
|
||||
}
|
||||
|
||||
function unresolved(provider: ModelProvider, key: string, cause: string, repair: string): ModelPolicyError {
|
||||
return new ModelPolicyError({
|
||||
reason: 'native_provider_unresolved',
|
||||
problem: `gstack cannot tell which ${PROVIDER_LABEL[provider]} endpoint your native settings use`,
|
||||
cause,
|
||||
key,
|
||||
source: 'native',
|
||||
repair: [repair, 'or name the model explicitly for this request'],
|
||||
docs: DOCS_PROVIDER,
|
||||
});
|
||||
}
|
||||
|
||||
const CLAUDE_FLAGS = ['CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY'] as const;
|
||||
|
||||
export interface NativeSettingsOptions {
|
||||
env?: StateRootEnv;
|
||||
platform?: NodeJS.Platform;
|
||||
cwd?: string;
|
||||
claudeManagedDir?: string;
|
||||
codexSystemConfig?: string | null;
|
||||
}
|
||||
|
||||
export interface CustomProviderDetection {
|
||||
custom: boolean;
|
||||
signals: string[];
|
||||
scanned: string[];
|
||||
unscanned: string[];
|
||||
}
|
||||
|
||||
export const CUSTOM_PROVIDER_UNSCANNED: Readonly<Record<ModelProvider, readonly string[]>> = Object.freeze({
|
||||
anthropic: Object.freeze(['server-managed settings from the claude.ai console', 'MDM or OS policy (macOS profile, Windows registry)', 'a --settings flag passed to claude']),
|
||||
openai: Object.freeze(['cloud-managed Codex config', '-c, --config or --profile flags passed to codex']),
|
||||
});
|
||||
|
||||
function claudeManagedDir(platform: NodeJS.Platform): string {
|
||||
if (platform === 'darwin') return '/Library/Application Support/ClaudeCode';
|
||||
if (platform === 'win32') return 'C:\\Program Files\\ClaudeCode';
|
||||
return '/etc/claude-code';
|
||||
}
|
||||
|
||||
function gitRoots(cwd: string): string[] {
|
||||
for (let dir = cwd; ; dir = path.dirname(dir)) {
|
||||
const dotGit = path.join(dir, '.git');
|
||||
let stat: fs.Stats | null = null;
|
||||
try {
|
||||
stat = fs.lstatSync(dotGit);
|
||||
} catch {
|
||||
stat = null;
|
||||
}
|
||||
if (stat?.isDirectory()) return [dir];
|
||||
if (stat?.isFile()) {
|
||||
try {
|
||||
const gitDir = path.resolve(dir, fs.readFileSync(dotGit, 'utf-8').replace(/^gitdir:\s*/, '').trim());
|
||||
const commonDir = path.resolve(gitDir, fs.readFileSync(path.join(gitDir, 'commondir'), 'utf-8').trim());
|
||||
return path.basename(commonDir) === '.git' ? [dir, path.dirname(commonDir)] : [dir];
|
||||
} catch {
|
||||
return [dir];
|
||||
}
|
||||
}
|
||||
if (path.dirname(dir) === dir) return [];
|
||||
}
|
||||
}
|
||||
|
||||
function claudeSettingsFiles(env: StateRootEnv, platform: NodeJS.Platform, cwd: string, managedDir: string): string[] {
|
||||
const files = [
|
||||
path.join(env.CLAUDE_CONFIG_DIR || path.join(homeDir(env, platform), '.claude'), 'settings.json'),
|
||||
path.join(cwd, '.claude', 'settings.json'),
|
||||
path.join(cwd, '.claude', 'settings.local.json'),
|
||||
...gitRoots(cwd).map(root => path.join(root, '.claude', 'settings.local.json')),
|
||||
path.join(managedDir, 'managed-settings.json'),
|
||||
];
|
||||
const dropIns = path.join(managedDir, 'managed-settings.d');
|
||||
try {
|
||||
files.push(...fs.readdirSync(dropIns).filter(name => name.endsWith('.json') && !name.startsWith('.')).sort().map(name => path.join(dropIns, name)));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw unresolved('anthropic', dropIns, `${dropIns} could not be read`, `fix the permissions of ${dropIns}`);
|
||||
}
|
||||
}
|
||||
return [...new Set(files)];
|
||||
}
|
||||
|
||||
function claudeSettingsEnv(file: string): Record<string, unknown> | null {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(file, 'utf-8');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw unresolved('anthropic', file, `${file} could not be read`, `fix the permissions of ${file}`);
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw unresolved('anthropic', file, `${file} is not valid JSON`, `fix the JSON in ${file}`);
|
||||
}
|
||||
const block = (parsed as { env?: unknown } | null)?.env;
|
||||
if (block === undefined) return {};
|
||||
if (typeof block !== 'object' || block === null || Array.isArray(block)) {
|
||||
throw unresolved('anthropic', file, `the env block in ${file} is not an object`, `fix the env block in ${file}`);
|
||||
}
|
||||
for (const name of [...CLAUDE_FLAGS, 'ANTHROPIC_BASE_URL']) {
|
||||
const value = (block as Record<string, unknown>)[name];
|
||||
if (value !== undefined && typeof value !== 'string') {
|
||||
throw unresolved('anthropic', `${file} env.${name}`, `env.${name} in ${file} is not a string`, `fix env.${name} in ${file}`);
|
||||
}
|
||||
}
|
||||
return block as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function claudeSignals(env: StateRootEnv, platform: NodeJS.Platform, cwd: string, managedDir: string): { signals: string[]; scanned: string[] } {
|
||||
const sources: Array<[string, Record<string, unknown>]> = [['environment', env]];
|
||||
const scanned = ['environment'];
|
||||
for (const file of claudeSettingsFiles(env, platform, cwd, managedDir)) {
|
||||
scanned.push(file);
|
||||
const block = claudeSettingsEnv(file);
|
||||
if (block) sources.push([file, block]);
|
||||
}
|
||||
const signals: string[] = [];
|
||||
for (const [origin, values] of sources) {
|
||||
for (const flag of CLAUDE_FLAGS) {
|
||||
if (truthy(values[flag])) signals.push(`${flag} (${origin})`);
|
||||
}
|
||||
if (nonDefaultUrl(values.ANTHROPIC_BASE_URL, 'https://api.anthropic.com')) signals.push(`ANTHROPIC_BASE_URL (${origin})`);
|
||||
}
|
||||
return { signals, scanned };
|
||||
}
|
||||
|
||||
function readToml(file: string): Record<string, unknown> | null {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(file, 'utf-8');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw unresolved('openai', file, `${file} could not be read`, `fix the permissions of ${file}`);
|
||||
}
|
||||
try {
|
||||
return Bun.TOML.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw unresolved('openai', file, `${file} is not valid TOML`, `fix the TOML in ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
function codexSignals(env: StateRootEnv, platform: NodeJS.Platform, systemConfig: string | null): { signals: string[]; scanned: string[] } {
|
||||
const codexHome = env.CODEX_HOME || path.join(homeDir(env, platform), '.codex');
|
||||
const signals: string[] = [];
|
||||
if (nonDefaultUrl(env.OPENAI_BASE_URL, 'https://api.openai.com/v1')) signals.push('OPENAI_BASE_URL (environment)');
|
||||
if (!path.isAbsolute(codexHome)) {
|
||||
throw unresolved('openai', 'CODEX_HOME', `CODEX_HOME '${sanitize(codexHome)}' is not an absolute path`, 'set CODEX_HOME to an absolute path');
|
||||
}
|
||||
const configPath = path.join(codexHome, 'config.toml');
|
||||
const scanned = ['environment', configPath, ...(systemConfig ? [systemConfig] : [])];
|
||||
const user = readToml(configPath) ?? {};
|
||||
const layers: Array<[string, Record<string, unknown>]> = [];
|
||||
if (user.profile !== undefined) {
|
||||
const profiles = user.profiles as Record<string, unknown> | undefined;
|
||||
const selected = typeof user.profile === 'string' && typeof profiles === 'object' && profiles !== null ? profiles[user.profile] : undefined;
|
||||
if (typeof selected !== 'object' || selected === null || Array.isArray(selected)) {
|
||||
throw unresolved('openai', `${configPath} profile`, `the active profile in ${configPath} is not a defined [profiles.<name>] table`, `define the profile or remove profile from ${configPath}`);
|
||||
}
|
||||
layers.push([`${configPath} [profiles.${sanitize(user.profile as string)}]`, selected as Record<string, unknown>]);
|
||||
}
|
||||
layers.push([configPath, user]);
|
||||
if (systemConfig) layers.push([systemConfig, readToml(systemConfig) ?? {}]);
|
||||
for (const key of ['model_provider', 'openai_base_url']) {
|
||||
const winner = layers.find(([, layer]) => layer[key] !== undefined);
|
||||
if (!winner) continue;
|
||||
const [origin, layer] = winner;
|
||||
const value = layer[key];
|
||||
if (typeof value !== 'string') {
|
||||
throw unresolved('openai', `${origin} ${key}`, `${key} in ${origin} is not a string`, `fix ${key} in ${origin}`);
|
||||
}
|
||||
if (key === 'model_provider' && value !== 'openai') signals.push(`model_provider "${sanitize(value)}" (${origin})`);
|
||||
if (key === 'openai_base_url' && nonDefaultUrl(value, 'https://api.openai.com/v1')) signals.push(`openai_base_url (${origin})`);
|
||||
}
|
||||
return { signals, scanned };
|
||||
}
|
||||
|
||||
export function detectCustomProvider(provider: ModelProvider, opts: NativeSettingsOptions = {}): CustomProviderDetection {
|
||||
const env = opts.env ?? process.env;
|
||||
const platform = opts.platform ?? process.platform;
|
||||
const { signals, scanned } = provider === 'anthropic'
|
||||
? claudeSignals(env, platform, path.resolve(opts.cwd ?? process.cwd()), opts.claudeManagedDir ?? claudeManagedDir(platform))
|
||||
: codexSignals(env, platform, opts.codexSystemConfig !== undefined ? opts.codexSystemConfig : platform === 'win32' ? null : '/etc/codex/config.toml');
|
||||
return { custom: signals.length > 0, signals, scanned, unscanned: [...CUSTOM_PROVIDER_UNSCANNED[provider]] };
|
||||
}
|
||||
|
||||
function sourceLabel(provider: ModelProvider, tier: ModelTier, entry: ModelCatalogEntry): string {
|
||||
return `gstack catalog ${tier}/${provider} (verified ${entry.verifiedAt})`;
|
||||
}
|
||||
|
||||
function checkedSource(provider: ModelProvider, raw: string, source: SelectedModel['source']): { model: string; source: SelectedModel['source'] } {
|
||||
const model = raw.trim();
|
||||
if (isValidModelId(provider, model)) return { model, source };
|
||||
const repair = source.kind === 'env'
|
||||
? [`unset ${source.name}`, `export ${source.name}=<model-id>`]
|
||||
: ['name a valid model ID for this request, or omit it to use the configured policy'];
|
||||
throw new ModelPolicyError({
|
||||
reason: 'invalid_model_id',
|
||||
problem: `the ${provider} model '${sanitize(raw)}' from ${source.label} is not a valid model ID`,
|
||||
cause: `use ${modelIdRule(provider)}; ${source.label} outranks gstack's tier settings, so changing a tier will not help`,
|
||||
key: source.kind === 'env' ? source.name : 'request',
|
||||
source: source.kind,
|
||||
repair,
|
||||
docs: DOCS_SELECTION,
|
||||
});
|
||||
}
|
||||
|
||||
function hostTool(provider: ModelProvider): string {
|
||||
return provider === 'anthropic' ? 'Claude Code' : 'Codex';
|
||||
}
|
||||
|
||||
export function resolvePlanReviewModel(opts: {
|
||||
provider: ModelProvider;
|
||||
requestedModel?: string;
|
||||
env?: StateRootEnv;
|
||||
platform?: NodeJS.Platform;
|
||||
cwd?: string;
|
||||
claudeManagedDir?: string;
|
||||
codexSystemConfig?: string | null;
|
||||
config?: ModelPolicyConfig;
|
||||
}): ModelSelection {
|
||||
const env = opts.env ?? process.env;
|
||||
const platform = opts.platform ?? process.platform;
|
||||
const config = opts.config ?? readModelPolicyConfig({ env, platform });
|
||||
const { provider } = opts;
|
||||
const tierFrom = config.planReviewTier;
|
||||
const tier = tierFrom.value;
|
||||
const base = { role: 'plan-review' as const, provider, tier, tierFrom, catalog: tier === 'host' ? null : catalogEntry(provider, tier) };
|
||||
if (opts.requestedModel !== undefined) {
|
||||
const { model, source } = checkedSource(provider, opts.requestedModel, { kind: 'request', label: 'explicit request' });
|
||||
return { ...base, status: 'selected', requestedModel: model, source };
|
||||
}
|
||||
const envName = PROVIDER_ENV_OVERRIDES[provider];
|
||||
const envValue = env[envName];
|
||||
if (envValue) {
|
||||
const { model, source } = checkedSource(provider, envValue, { kind: 'env', name: envName, label: `${envName} environment variable` });
|
||||
return { ...base, status: 'selected', requestedModel: model, source };
|
||||
}
|
||||
if (tier === 'host') {
|
||||
return {
|
||||
...base,
|
||||
status: 'delegated-host',
|
||||
requestedModel: null,
|
||||
source: { kind: 'host', reason: 'host-mode', key: 'plan_review_tier', label: `plan_review_tier=host: ${hostTool(provider)}'s own settings choose the model` },
|
||||
};
|
||||
}
|
||||
const overrideKey = MODEL_OVERRIDE_KEYS[provider][tier];
|
||||
const override = config.overrides[provider][tier];
|
||||
if (override) {
|
||||
return { ...base, status: 'selected', requestedModel: override, source: { kind: 'config', key: overrideKey, path: config.path, label: `${overrideKey} in ${config.path}` } };
|
||||
}
|
||||
const entry = catalogEntry(provider, tier);
|
||||
const custom = detectCustomProvider(provider, { ...opts, env, platform });
|
||||
if (custom.custom) {
|
||||
throw new ModelPolicyError({
|
||||
reason: 'custom_provider_requires_model',
|
||||
problem: `no ${tier} plan-review model is set for your custom ${PROVIDER_LABEL[provider]} endpoint`,
|
||||
cause: `custom routing is active (${custom.signals.join(', ')}), so gstack will not send the public catalog ID ${entry.model} to it`,
|
||||
key: custom.signals[0] ?? null,
|
||||
source: 'native',
|
||||
repair: [`gstack-config set ${overrideKey} <model-id-your-endpoint-serves>`, `export ${envName}=<model-id-your-endpoint-serves>`, 'gstack-config set plan_review_tier host'],
|
||||
docs: DOCS_PROVIDER,
|
||||
});
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: 'selected',
|
||||
requestedModel: entry.model,
|
||||
source: { kind: 'catalog', catalogVersion: MODEL_CATALOG_VERSION, verifiedAt: entry.verifiedAt, sourceUrl: entry.sourceUrl, label: sourceLabel(provider, tier, entry) },
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveImplementationModels(opts: {
|
||||
provider?: ModelProvider;
|
||||
host?: ModelHost;
|
||||
requestedModel?: string;
|
||||
env?: StateRootEnv;
|
||||
platform?: NodeJS.Platform;
|
||||
cwd?: string;
|
||||
claudeManagedDir?: string;
|
||||
codexSystemConfig?: string | null;
|
||||
config?: ModelPolicyConfig;
|
||||
} = {}): ModelSelection[] {
|
||||
const env = opts.env ?? process.env;
|
||||
const platform = opts.platform ?? process.platform;
|
||||
const providers: ModelProvider[] = opts.provider ? [opts.provider]
|
||||
: opts.host === 'claude-code' ? ['anthropic']
|
||||
: opts.host === 'codex' ? ['openai']
|
||||
: [...MODEL_PROVIDERS];
|
||||
if (opts.requestedModel !== undefined && providers.length !== 1) {
|
||||
throw new ModelPolicyError({
|
||||
reason: 'provider_required',
|
||||
problem: 'an explicit implementation model needs a provider',
|
||||
cause: 'the host does not identify one provider, and gstack does not infer a provider from a model name',
|
||||
key: 'request',
|
||||
source: 'request',
|
||||
repair: ['pass the provider (anthropic or openai) with the requested model'],
|
||||
docs: DOCS_SELECTION,
|
||||
});
|
||||
}
|
||||
const config = opts.config ?? readModelPolicyConfig({ env, platform });
|
||||
const tierFrom = config.implementationTier;
|
||||
const tier = tierFrom.value;
|
||||
return providers.map((provider): ModelSelection => {
|
||||
const entry = catalogEntry(provider, tier);
|
||||
const base = { role: 'implementation' as const, provider, tier, tierFrom, catalog: entry };
|
||||
if (opts.requestedModel !== undefined) {
|
||||
const { model, source } = checkedSource(provider, opts.requestedModel, { kind: 'request', label: 'explicit request' });
|
||||
return { ...base, status: 'selected', requestedModel: model, source };
|
||||
}
|
||||
const overrideKey = MODEL_OVERRIDE_KEYS[provider][tier];
|
||||
const override = config.overrides[provider][tier];
|
||||
if (override) {
|
||||
return { ...base, status: 'selected', requestedModel: override, source: { kind: 'config', key: overrideKey, path: config.path, label: `${overrideKey} in ${config.path}` } };
|
||||
}
|
||||
const custom = detectCustomProvider(provider, { ...opts, env, platform });
|
||||
if (custom.custom) {
|
||||
throw new ModelPolicyError({
|
||||
reason: 'custom_provider_requires_model',
|
||||
problem: `no ${tier} implementation model is set for your custom ${PROVIDER_LABEL[provider]} endpoint`,
|
||||
cause: `custom routing is active (${custom.signals.join(', ')}), so gstack will not recommend the public catalog ID ${entry.model} for it`,
|
||||
key: custom.signals[0] ?? null,
|
||||
source: 'native',
|
||||
repair: [`gstack-config set ${overrideKey} <model-id-your-endpoint-serves>`, 'or name the implementation model explicitly for this request'],
|
||||
docs: DOCS_PROVIDER,
|
||||
});
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: 'selected',
|
||||
requestedModel: entry.model,
|
||||
source: { kind: 'catalog', catalogVersion: MODEL_CATALOG_VERSION, verifiedAt: entry.verifiedAt, sourceUrl: entry.sourceUrl, label: sourceLabel(provider, tier, entry) },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function describeSelection(selection: ModelSelection): string {
|
||||
const model = selection.status === 'selected' ? selection.requestedModel : 'host-controlled (unknown until it runs)';
|
||||
return `${selection.role} via ${selection.provider}: ${model} [tier ${selection.tier}; from ${selection.source.label}]`;
|
||||
}
|
||||
|
||||
export function selectionRepair(selection: ModelSelection): string[] {
|
||||
const { source, provider } = selection;
|
||||
if (source.kind === 'request') return ['name a different model for this request, or omit it to use the configured policy'];
|
||||
if (source.kind === 'env') return [`unset ${source.name}`, `export ${source.name}=<model-id>`];
|
||||
if (source.kind === 'config') return [`gstack-config set ${source.key} <model-id>`, `gstack-config unset ${source.key}`];
|
||||
const tierKey = selection.role === 'plan-review' ? 'plan_review_tier' : 'implementation_tier';
|
||||
if (source.kind === 'host') {
|
||||
const native = provider === 'anthropic' ? 'choose the model in Claude Code settings' : 'set model in ${CODEX_HOME:-~/.codex}/config.toml';
|
||||
return [native, `export ${PROVIDER_ENV_OVERRIDES[provider]}=<model-id>`, 'gstack-config unset plan_review_tier'];
|
||||
}
|
||||
const tier = selection.tier as ModelTier;
|
||||
const other = tier === 'frontier' ? 'smart' : 'frontier';
|
||||
return [
|
||||
`gstack-config set ${MODEL_OVERRIDE_KEYS[provider][tier]} <model-id>`,
|
||||
`gstack-config set ${tierKey} ${other}`,
|
||||
...(selection.role === 'plan-review' ? ['gstack-config set plan_review_tier host'] : []),
|
||||
];
|
||||
}
|
||||
|
||||
export function modelPolicyCommands() {
|
||||
return {
|
||||
useSmart: 'gstack-config set plan_review_tier smart',
|
||||
useHost: 'gstack-config set plan_review_tier host',
|
||||
restoreDefault: 'gstack-config unset plan_review_tier',
|
||||
overrideExample: (provider: ModelProvider, tier: ModelTier) => `gstack-config set ${MODEL_OVERRIDE_KEYS[provider][tier]} <model-id>`,
|
||||
unsetOverride: (provider: ModelProvider, tier: ModelTier) => `gstack-config unset ${MODEL_OVERRIDE_KEYS[provider][tier]}`,
|
||||
};
|
||||
}
|
||||
@@ -357,56 +357,27 @@ review. The user turns this off only by asking explicitly
|
||||
|
||||
```bash
|
||||
|
||||
# Codex preflight: one block (functions sourced here don't persist to later blocks).
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
|
||||
_gstack_helper_error=""
|
||||
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
|
||||
if [ "$_CODEX_CFG" = "disabled" ]; then
|
||||
_CODEX_MODE="disabled"
|
||||
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
_CODEX_MODE="under_codex"
|
||||
elif ! command -v codex >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
|
||||
elif [ -n "$_gstack_helper_error" ]; then
|
||||
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
|
||||
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
|
||||
else
|
||||
# The free sandbox check runs before the paid model probe. Probe code 2 means
|
||||
# the CLI cannot execute at all, a different fix from an unusable model.
|
||||
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
|
||||
if [ "$_CODEX_MP" -eq 3 ]; then
|
||||
_CODEX_MODE="sandbox_unavailable"
|
||||
elif [ "$_CODEX_MP" -eq 2 ]; then
|
||||
_CODEX_MODE="broken_install"
|
||||
elif [ "$_CODEX_MP" -eq 4 ]; then
|
||||
_CODEX_MODE="quota_exhausted"
|
||||
elif [ "$_CODEX_MP" -ne 0 ]; then
|
||||
_CODEX_MODE="model_unusable"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
|
||||
_CODEX_MODE="unverified"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
|
||||
_CODEX_MODE="unverified (rate_limited)"
|
||||
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
|
||||
if [ "$_OUTSIDE_CFG" = disabled ]; then
|
||||
echo 'CODEX_MODE: disabled'
|
||||
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
|
||||
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
|
||||
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
|
||||
else
|
||||
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
|
||||
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
|
||||
fi
|
||||
exit 78
|
||||
fi
|
||||
); then
|
||||
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
|
||||
else
|
||||
echo 'CODEX_MODE: under_current_harness'
|
||||
fi
|
||||
echo "CODEX_MODE: $_CODEX_MODE"
|
||||
```
|
||||
|
||||
Branch on the echoed `CODEX_MODE`:
|
||||
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the reviewer invocation; record disabled coverage as directed below; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
|
||||
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
|
||||
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
|
||||
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and construct the prompt below, then follow **Native fallback**. Conflicting inherited harness markers are not grounds to guess another provider.
|
||||
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
|
||||
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
|
||||
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
|
||||
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
|
||||
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
|
||||
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
|
||||
|
||||
**Outcome routing:** Follow the row for the current result. After an invocation, route its result
|
||||
again. Leave only after recording disabled/unavailable coverage, or after
|
||||
@@ -505,8 +476,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -993,7 +993,7 @@ else
|
||||
fi
|
||||
```
|
||||
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
|
||||
|
||||
Declined: skip both voices. Non-ready: retain the repair notice, use only the native voice, and record `outside_status: unavailable` even if it succeeds. The invocation rechecks the harness before spawning.
|
||||
|
||||
@@ -1050,8 +1050,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -296,56 +296,27 @@ review. The user turns this off only by asking explicitly
|
||||
|
||||
```bash
|
||||
|
||||
# Codex preflight: one block (functions sourced here don't persist to later blocks).
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
|
||||
_gstack_helper_error=""
|
||||
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
|
||||
if [ "$_CODEX_CFG" = "disabled" ]; then
|
||||
_CODEX_MODE="disabled"
|
||||
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
_CODEX_MODE="under_codex"
|
||||
elif ! command -v codex >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
|
||||
elif [ -n "$_gstack_helper_error" ]; then
|
||||
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
|
||||
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
|
||||
else
|
||||
# The free sandbox check runs before the paid model probe. Probe code 2 means
|
||||
# the CLI cannot execute at all, a different fix from an unusable model.
|
||||
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
|
||||
if [ "$_CODEX_MP" -eq 3 ]; then
|
||||
_CODEX_MODE="sandbox_unavailable"
|
||||
elif [ "$_CODEX_MP" -eq 2 ]; then
|
||||
_CODEX_MODE="broken_install"
|
||||
elif [ "$_CODEX_MP" -eq 4 ]; then
|
||||
_CODEX_MODE="quota_exhausted"
|
||||
elif [ "$_CODEX_MP" -ne 0 ]; then
|
||||
_CODEX_MODE="model_unusable"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
|
||||
_CODEX_MODE="unverified"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
|
||||
_CODEX_MODE="unverified (rate_limited)"
|
||||
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
|
||||
if [ "$_OUTSIDE_CFG" = disabled ]; then
|
||||
echo 'CODEX_MODE: disabled'
|
||||
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
|
||||
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
|
||||
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
|
||||
else
|
||||
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
|
||||
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
|
||||
fi
|
||||
exit 78
|
||||
fi
|
||||
); then
|
||||
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
|
||||
else
|
||||
echo 'CODEX_MODE: under_current_harness'
|
||||
fi
|
||||
echo "CODEX_MODE: $_CODEX_MODE"
|
||||
```
|
||||
|
||||
Branch on the echoed `CODEX_MODE`:
|
||||
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip this section entirely; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
|
||||
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
|
||||
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
|
||||
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider.
|
||||
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
|
||||
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
|
||||
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
|
||||
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
|
||||
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
|
||||
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
|
||||
|
||||
**Disabled is a terminal branch for this section.** If the preflight prints
|
||||
`CODEX_MODE: disabled`, persist `outside_status: disabled` with the guarded
|
||||
@@ -439,8 +410,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
@@ -483,7 +453,7 @@ Immediately before dispatching, check the preflight result again. On
|
||||
do not dispatch. Otherwise, use this fallback for missing/broken CLI, failed
|
||||
authentication/model selection, a failed preflight, or a failed outside invocation.
|
||||
The disabled branch never reaches this fallback.
|
||||
On `CODEX_MODE: under_codex`, report the setup repair and
|
||||
On `CODEX_MODE: under_current_harness`, report the setup repair and
|
||||
`outside_status: unavailable`, run no outside CLI, and use the native subagent below.
|
||||
A native result never supplies outside coverage.
|
||||
|
||||
|
||||
@@ -799,56 +799,27 @@ review. The user turns this off only by asking explicitly
|
||||
|
||||
```bash
|
||||
|
||||
# Codex preflight: one block (functions sourced here don't persist to later blocks).
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || echo off)
|
||||
_CODEX_CFG=$(~/.claude/skills/gstack/bin/gstack-config get codex_reviews 2>/dev/null || echo enabled)
|
||||
_gstack_helper_error=""
|
||||
. ~/.claude/skills/gstack/bin/gstack-codex-probe 2>/dev/null || _gstack_helper_error="${_gstack_helper_error:-gstack: cannot load gstack-codex-probe; re-run ./setup. https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#sourced-helper-location}"
|
||||
if [ "$_CODEX_CFG" = "disabled" ]; then
|
||||
_CODEX_MODE="disabled"
|
||||
elif { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
_CODEX_MODE="under_codex"
|
||||
elif ! command -v codex >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_installed"; _gstack_codex_log_event "codex_cli_missing" 2>/dev/null || true
|
||||
elif [ -n "$_gstack_helper_error" ]; then
|
||||
_CODEX_MODE="helper_unavailable"; echo "$_gstack_helper_error"
|
||||
elif ! _gstack_codex_auth_probe >/dev/null 2>&1; then
|
||||
_CODEX_MODE="not_authed"; _gstack_codex_log_event "codex_auth_failed" 2>/dev/null || true
|
||||
else
|
||||
# The free sandbox check runs before the paid model probe. Probe code 2 means
|
||||
# the CLI cannot execute at all, a different fix from an unusable model.
|
||||
_CODEX_MP=0; _gstack_codex_sandbox_preflight || _CODEX_MP=3
|
||||
[ "$_CODEX_MP" -ne 0 ] || { _gstack_codex_model_probe; _CODEX_MP=$?; }
|
||||
if [ "$_CODEX_MP" -eq 3 ]; then
|
||||
_CODEX_MODE="sandbox_unavailable"
|
||||
elif [ "$_CODEX_MP" -eq 2 ]; then
|
||||
_CODEX_MODE="broken_install"
|
||||
elif [ "$_CODEX_MP" -eq 4 ]; then
|
||||
_CODEX_MODE="quota_exhausted"
|
||||
elif [ "$_CODEX_MP" -ne 0 ]; then
|
||||
_CODEX_MODE="model_unusable"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = inconclusive ]; then
|
||||
_CODEX_MODE="unverified"
|
||||
elif [ "${_GSTACK_CODEX_PROBE_STATE:-}" = rate_limited ]; then
|
||||
_CODEX_MODE="unverified (rate_limited)"
|
||||
_OUTSIDE_CFG=$("$HOME/.claude/skills/gstack/bin/gstack-config" get codex_reviews 2>/dev/null || echo enabled)
|
||||
if [ "$_OUTSIDE_CFG" = disabled ]; then
|
||||
echo 'CODEX_MODE: disabled'
|
||||
elif ( # GSTACK_ACTIVE_HOST names the harness, never the model.
|
||||
if { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage.' >&2
|
||||
if { [ -n "${CLAUDECODE:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = claude ]; } && { [ -n "${CODEX_THREAD_ID:-}" ] || [ -n "${CODEX_SANDBOX:-}" ] || [ "${GSTACK_ACTIVE_HOST:-}" = codex ]; }; then
|
||||
echo 'Inherited harness markers conflict. Run setup --host <actual-harness> (claude or codex); do not guess a replacement provider.' >&2
|
||||
else
|
||||
_CODEX_MODE="ready"; _gstack_codex_version_check 2>/dev/null || true
|
||||
echo 'Repair installed skills: run setup --host codex from your gstack checkout.' >&2
|
||||
fi
|
||||
exit 78
|
||||
fi
|
||||
); then
|
||||
if command -v codex >/dev/null 2>&1; then echo 'CODEX_MODE: ready'; else echo 'CODEX_MODE: not_installed'; fi
|
||||
else
|
||||
echo 'CODEX_MODE: under_current_harness'
|
||||
fi
|
||||
echo "CODEX_MODE: $_CODEX_MODE"
|
||||
```
|
||||
|
||||
Branch on the echoed `CODEX_MODE`:
|
||||
- **`disabled`** — the user turned Codex reviews off (`codex_reviews=disabled`). Skip the reviewer invocation; record disabled coverage as directed below; do NOT fall back to a Claude subagent — disabled means no extra review step. Print: "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`."
|
||||
- **`helper_unavailable`** — the helper could not load; relay the line above (cause and fix). Fall back to the Claude subagent path.
|
||||
- **`not_installed`** — Codex CLI absent. Print: "Codex not installed; outside coverage unavailable. Install: `npm install -g @openai/codex`." Fall back to the Claude subagent path.
|
||||
- **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and construct the prompt below, then follow **Native fallback**. Conflicting inherited harness markers are not grounds to guess another provider.
|
||||
- **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Fall back to the Claude subagent path.
|
||||
- **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Fall back to the Claude subagent path.
|
||||
- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=<supported-model>` or config.toml `model`); never substitute a model. Fall back to the Claude subagent path. The ~10s round trip is cached for 1h.
|
||||
- **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Fall back to the Claude subagent path.
|
||||
- **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Fall back to the Claude subagent path.
|
||||
- **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer. Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled. Never substitute another external provider.
|
||||
|
||||
**Outcome routing:** Pick exactly one row from this table, finish that row's
|
||||
steps, then leave Outside Voice. Missing reviewer coverage is non-blocking;
|
||||
@@ -944,8 +915,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 300 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
@@ -1474,6 +1444,8 @@ without adding or strengthening them in the question or descriptions. A test
|
||||
required before editing one function does not make every independent lane wait.
|
||||
A next-step answer approves no implementation change.
|
||||
|
||||
When the user chooses C: **Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
|
||||
|
||||
## Learning hooks
|
||||
|
||||
In finish step 6, keep the working plan/approvals fixed. Review operational learnings
|
||||
|
||||
@@ -592,6 +592,8 @@ without adding or strengthening them in the question or descriptions. A test
|
||||
required before editing one function does not make every independent lane wait.
|
||||
A next-step answer approves no implementation change.
|
||||
|
||||
When the user chooses C: {{IMPLEMENTATION_MODEL_HANDOFF}}
|
||||
|
||||
## Learning hooks
|
||||
|
||||
In finish step 6, keep the working plan/approvals fixed. Review operational learnings
|
||||
|
||||
+3
-3
@@ -650,7 +650,7 @@ Sanitize every query before it leaves the machine: strip hostnames, IPs, file pa
|
||||
|
||||
## Step 4: Critical pass (core review)
|
||||
|
||||
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
|
||||
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
|
||||
Step 4 is read-only: defer charters, setup and probes to Step 4.7.
|
||||
|
||||
From the installed /review SKILL.md's directory, choose one path:
|
||||
@@ -658,9 +658,9 @@ From the installed /review SKILL.md's directory, choose one path:
|
||||
- If the caller directory is prefixed `gstack-review`, use `../gstack-qa/sections/exploratory.md` instead and read it in full.
|
||||
- If neither layout applies, report an unresolved QA installation as a setup blocker; do not guess another path.
|
||||
Use this host's installation, never the product tree. If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.
|
||||
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
|
||||
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
|
||||
|
||||
Resolve QA's `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory.
|
||||
QA's `sections/...` and `templates/...` paths resolve from installed QA SKILL.md, not the caller or product directory.
|
||||
|
||||
Apply both checklist passes in order: CRITICAL, then INFORMATIONAL. Respect its suppressions.
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { readFileSync, writeFileSync, appendFileSync } from 'node:fs';
|
||||
import { collectFreshnessSources, type Fetcher } from '../lib/model-policy-freshness-sources';
|
||||
import { assessFreshness, freshnessCatalogIdentity, renderFreshnessReport, type FreshnessObservation } from '../lib/model-policy-freshness';
|
||||
import { freshnessGitHub } from '../lib/model-policy-freshness-github';
|
||||
import { publishFreshness, type FreshnessGitHub } from '../lib/model-policy-freshness-publication';
|
||||
|
||||
export async function runModelPolicyFreshness(options: {
|
||||
env?: NodeJS.ProcessEnv; fetcher?: Fetcher; github?: FreshnessGitHub; now?: () => string; publish?: boolean;
|
||||
}) {
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? (() => new Date().toISOString());
|
||||
const startedAt = now();
|
||||
const catalog = freshnessCatalogIdentity(readFileSync(new URL('../lib/model-catalog.ts', import.meta.url), 'utf8'));
|
||||
const sources = await collectFreshnessSources(options.fetcher);
|
||||
const observation: FreshnessObservation = {
|
||||
catalog,
|
||||
checkedAt: now(), run: { id: env.GITHUB_RUN_ID ?? '0', attempt: Number(env.GITHUB_RUN_ATTEMPT ?? '1'), startedAt, headSha: env.GITHUB_SHA ?? '0'.repeat(40) }, sources,
|
||||
};
|
||||
let report = assessFreshness(observation);
|
||||
let publication;
|
||||
if (options.publish) {
|
||||
if (!env.GITHUB_REF) throw new Error('publication-ref-missing');
|
||||
if (env.GITHUB_REF === 'refs/heads/main' && (!env.GITHUB_REPOSITORY || !env.GH_TOKEN || !env.GITHUB_RUN_ID || !env.GITHUB_SHA)) throw new Error('github-publication-prerequisites-missing');
|
||||
try {
|
||||
const github = options.github ?? (env.GITHUB_REF === 'refs/heads/main' ? freshnessGitHub(env.GITHUB_REPOSITORY!, env.GH_TOKEN!, options.fetcher) : null);
|
||||
publication = github ? await publishFreshness(observation, github, env.GITHUB_REF) : { action: 'report-only' as const };
|
||||
if ('report' in publication && publication.report) report = publication.report;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error && /^[a-z0-9: /.-]{1,160}$/i.test(error.message) ? error.message : 'publication-unavailable';
|
||||
publication = { action: 'failed' as const, error: reason };
|
||||
report = { ...report, status: 'unknown/source-unavailable', complete: false, errors: [...report.errors, `publication:${reason}`] };
|
||||
}
|
||||
}
|
||||
return { schemaVersion: 1, observation, report, publication };
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = process.argv.slice(2);
|
||||
if (args.some(arg => !['--publish', '--help'].includes(arg))) { console.error('Usage: bun scripts/model-policy-freshness.ts [--publish]'); return 2; }
|
||||
if (args.includes('--help')) {
|
||||
console.log('Check four official public model recommendation/lifecycle sources without paid calls. Default: report-only. --publish requires main/default-branch identity and GITHUB_TOKEN-backed GH_TOKEN; no catalog or pin writes. JSON goes to stdout. Optional MODEL_FRESHNESS_REPORT writes the same artifact; GITHUB_STEP_SUMMARY receives the human report.');
|
||||
return 0;
|
||||
}
|
||||
const result = await runModelPolicyFreshness({ publish: args.includes('--publish') });
|
||||
const output = `${JSON.stringify(result, null, 2)}\n`;
|
||||
if (process.env.MODEL_FRESHNESS_REPORT) writeFileSync(process.env.MODEL_FRESHNESS_REPORT, output, { mode: 0o600 });
|
||||
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${renderFreshnessReport(result.report)}\n`);
|
||||
console.log(output.trimEnd());
|
||||
if (result.publication?.action.startsWith('rejected-') || result.publication?.action === 'failed') return 1;
|
||||
return result.report.status === 'unknown/source-unavailable' || result.report.status === 'stale' ? 1 : 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().then(code => { process.exitCode = code; }).catch(() => {
|
||||
console.error('Model-policy freshness or report delivery failed. Confirm the tracking issue state; do not infer freshness from this run. Check the workflow logs, official sources, issue permissions/state, and docs/troubleshooting.md#model-policy-freshness.');
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import * as path from 'node:path';
|
||||
import { discoverTemplates, includesSkill } from './discover-skills';
|
||||
import { externalSkillName, extractNameAndDescription } from './external-skill-names';
|
||||
import { getHostConfig } from '../hosts';
|
||||
import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const value = (flag: string): string => {
|
||||
@@ -155,12 +156,11 @@ for (const name of hostEntries.filter(name => name.startsWith('gstack-') && !sou
|
||||
|
||||
const old = path.join(namespace, 'gstack-claude');
|
||||
const nextSkill = path.join(namespace, 'gstack-claude-code');
|
||||
const renameFiles = ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts'];
|
||||
const runtimeSkill = path.join(runtime, 'SKILL.md');
|
||||
const runtimeBanner = !fs.existsSync(runtimeSkill) || fs.readFileSync(runtimeSkill, 'utf8').includes('<!-- AUTO-GENERATED from');
|
||||
const rename = owned(old) && (!exists(nextSkill) || owned(nextSkill))
|
||||
&& (!exists(runtime)?.isSymbolicLink() || linkIsOurs(runtime))
|
||||
&& runtimeBanner && renameFiles.every(rel => {
|
||||
&& runtimeBanner && CLAUDE_CODE_RUNTIME_FILES.every(rel => {
|
||||
const parent = path.join(runtime, path.dirname(rel));
|
||||
return exists(path.join(source, rel)) && (!exists(parent)?.isSymbolicLink() || linkIsOurs(parent));
|
||||
});
|
||||
@@ -172,7 +172,7 @@ if (rename) {
|
||||
for (const rel of ['SKILL.md', 'agents/openai.yaml']) checkAtomicCopy(path.join(next, rel), physicalNamespace, 'rename skill write', true);
|
||||
}
|
||||
if (exists(runtime) && physical(runtime) !== source) {
|
||||
for (const rel of renameFiles) {
|
||||
for (const rel of CLAUDE_CODE_RUNTIME_FILES) {
|
||||
const dest = path.join(runtime, rel);
|
||||
if (exists(dest) && physical(dest) === physical(path.join(source, rel))) continue;
|
||||
checkAtomicCopy(dest, physicalNamespace, 'rename runtime write');
|
||||
|
||||
@@ -4,6 +4,8 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ALL_MODEL_NAMES, resolveModel, type Model } from './models';
|
||||
import { ModelPolicyError, resolvePlanReviewModel, selectionRepair, type ModelSelection } from '../lib/model-policy';
|
||||
import { emitModelPolicyNotice } from '../lib/model-policy-notice';
|
||||
|
||||
export interface CodexGenerationModelResolution {
|
||||
model: Model;
|
||||
@@ -181,6 +183,45 @@ export function resolveCodexRuntimeModel(opts: {
|
||||
return { kind: opts.kind, model: CODEX_DEFAULT_MODEL, source: `gstack default (no model in ${stripControl(configPath)})` };
|
||||
}
|
||||
|
||||
export interface CodexPlanReviewSelection extends CodexRuntimeModelSelection {
|
||||
selection: ModelSelection;
|
||||
/** Source-aware repair for this record, never a lower-priority setting. */
|
||||
repair: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Role-bearing Codex selection: the plan-review policy ranks request, env,
|
||||
* tier config and catalog. Host mode keeps the legacy native resolver,
|
||||
* labelled as delegated so its origin stays honest.
|
||||
*/
|
||||
export function resolveCodexPlanReviewModel(opts: {
|
||||
kind: CodexInvocationKind;
|
||||
explicit?: string;
|
||||
env?: Record<string, string | undefined>;
|
||||
codexHome?: string;
|
||||
home?: string;
|
||||
/** Target repository; its project settings can route to a custom endpoint. */
|
||||
cwd?: string;
|
||||
}): CodexPlanReviewSelection {
|
||||
const env = opts.env ?? process.env;
|
||||
const selection = resolvePlanReviewModel({ provider: 'openai', requestedModel: opts.explicit || undefined, env, cwd: opts.cwd });
|
||||
const repair = stripControl(selectionRepair(selection).join('; '));
|
||||
if (selection.status === 'selected') {
|
||||
const model = selection.requestedModel;
|
||||
if (!CODEX_RUNTIME_MODEL_PATTERN.test(model)) {
|
||||
throw new Error(`Invalid Codex model '${sanitize(model)}' from ${sanitize(selection.source.label)}. ${repair}`);
|
||||
}
|
||||
return { kind: opts.kind, model, source: stripControl(selection.source.label), selection, repair };
|
||||
}
|
||||
const native = resolveCodexRuntimeModel({ kind: opts.kind, env, codexHome: opts.codexHome, home: opts.home });
|
||||
return { ...native, source: stripControl(`${selection.source.label}; ${native.source}`), selection, repair: repair || REPAIR };
|
||||
}
|
||||
|
||||
function policyFailure(error: unknown): string {
|
||||
if (!(error instanceof ModelPolicyError)) return (error as Error).message;
|
||||
return [`${error.problem} ${error.cause}`.trim(), ...error.repair.map(step => `Repair: ${step}`), `Docs: ${error.docs}`].join('\n');
|
||||
}
|
||||
|
||||
function readArg(name: string): string | undefined {
|
||||
const exact = process.argv.indexOf(name);
|
||||
if (exact >= 0) return process.argv[exact + 1];
|
||||
@@ -193,7 +234,15 @@ if (import.meta.main) {
|
||||
try {
|
||||
const runtime = readArg('--runtime');
|
||||
if (runtime !== undefined) {
|
||||
if (runtime !== 'exec' && runtime !== 'review') throw new Error('Usage: --runtime exec|review [--explicit <model>]');
|
||||
if (runtime !== 'exec' && runtime !== 'review') throw new Error('Usage: --runtime exec|review [--explicit <model>] [--role plan-review]');
|
||||
const role = readArg('--role');
|
||||
if (role !== undefined) {
|
||||
if (role !== 'plan-review') throw new Error('Usage: --role plan-review');
|
||||
const record = resolveCodexPlanReviewModel({ kind: runtime, explicit: readArg('--explicit'), cwd: readArg('--cwd') });
|
||||
emitModelPolicyNotice(record.selection);
|
||||
process.stdout.write(`${record.model}\t${record.source}\t${record.selection.tier}\t${record.repair}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
const selection = resolveCodexRuntimeModel({ kind: runtime, explicit: readArg('--explicit') });
|
||||
process.stdout.write(`${selection.model}\t${selection.source}\n`);
|
||||
process.exit(0);
|
||||
@@ -209,7 +258,7 @@ if (import.meta.main) {
|
||||
// so a hostile CODEX_HOME cannot smuggle tabs/newlines into the TSV contract.
|
||||
process.stdout.write(`${result.model}\t${stripControl(result.source)}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${(error as Error).message}\n`);
|
||||
process.stderr.write(`${policyFailure(error)}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -57,9 +57,11 @@ export const OPENAI_LITMUS_CHECKS = [
|
||||
export const CODEX_WEB_SEARCH_FLAG = `-c 'web_search="cached"'`;
|
||||
|
||||
/**
|
||||
* Default model for gstack-owned Codex invocations when nothing else chooses.
|
||||
* Historical no-role Codex default and the pinned eval ruler some tests import.
|
||||
* It does not follow the role catalog in lib/model-catalog.ts: plan-review
|
||||
* calls resolve their model through lib/model-policy.ts at invocation time.
|
||||
*
|
||||
* The runtime model is resolved per invocation kind by
|
||||
* A no-role runtime model is resolved per invocation kind by
|
||||
* `_gstack_codex_select_model exec|review` (bin/gstack-codex-probe, backed by
|
||||
* resolveCodexRuntimeModel in scripts/resolve-codex-generation-model.ts):
|
||||
* explicit request, then GSTACK_CODEX_MODEL, then Codex config.toml (`model`;
|
||||
|
||||
@@ -841,7 +841,7 @@ printf 'DESIGN_BRIEF=%s\\n' "$_DESIGN_BRIEF"
|
||||
Write the product brief to that path; remember its absolute path across fresh Bash calls. Neither voice inherits context: give both the same brief. Include its complete contents in the outside prompt file for Codex, along with the design-direction request below; substitute its shell-quoted absolute path for the literal <prepared-prompt-file> in the invocation. Keep your draft direction out of both prompts; give the native Agent its absolute path (the product brief's path, not the Codex prompt file). Never paste brief text into shell source.` : ''}
|
||||
|
||||
**Check ${outsideVoiceFor(ctx).label} availability:**
|
||||
${outsideVoicePreflight(ctx, { disabledBehavior: 'opt-in', acceptedOnly: isDesignConsultation })}
|
||||
${outsideVoicePreflight(ctx, { disabledBehavior: 'opt-in', acceptedOnly: isDesignConsultation, ...(isPlanDesignReview ? { role: 'plan-review' as const } : {}) })}
|
||||
|
||||
${isDesignConsultation ? 'Non-ready CLI: retain its repair notice and use only the native voice. The invocation deliberately rechecks the harness before spawning; native success never replaces external coverage.' : 'Declined: skip both voices. Non-ready: retain the repair notice, use only the native voice, and record `outside_status: unavailable` even if it succeeds. The invocation rechecks the harness before spawning.'}
|
||||
|
||||
@@ -853,7 +853,7 @@ Prompt (include the actual plan/product/frontend source context, not only file p
|
||||
|
||||
"${codexPrompt}"
|
||||
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, reasoningEffort, ...(isDesignConsultation ? { purpose: 'design-direction' as const } : {}) })}
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, reasoningEffort, ...(isDesignConsultation ? { purpose: 'design-direction' as const } : {}), ...(isPlanDesignReview ? { role: 'plan-review' as const } : {}) })}
|
||||
|
||||
2. **${outsideVoiceFor(ctx).nativeLabel} design subagent** (Agent tool, \`run_in_background: false\`; await its result):
|
||||
"${subagentPrompt}"
|
||||
|
||||
@@ -27,6 +27,7 @@ import { generateReviewDashboard, generatePlanFileReviewReport } from './review-
|
||||
import { generatePlanReviewApprovalCheck, generateExitPlanModeGate, generatePlanCompletionAuditShip, generatePlanCompletionGateShip, generatePlanCompletionAuditReview, generatePlanVerificationExec } from './plan-gates';
|
||||
import { generateAntiShortcutClause, generateSpecReviewLoop, generateBenefitsFrom } from './spec-review';
|
||||
import { generateCodexSecondOpinion, generateAdversarialStep, generateCodexPlanReview, generateCodexDocReview } from './outside-voice-steps';
|
||||
import { generateImplementationModelHandoff } from './plan-review';
|
||||
import { generateScopeDrift, generateCrossReviewDedup, generateSharedCodeReuse } from './review-scope';
|
||||
import { generateSlugEval, generateSlugSetup, generateBaseBranchDetect, generateDeployBootstrap, generateQAMethodology, generateCoAuthorTrailer, generateChangelogWorkflow, generateCodexWebSearchFlag, generateCodexModelConfigFlag, generateCodexReviewModelConfigFlag, generateClaudeModelFlag, generateSetupCommand } from './utility';
|
||||
import { generateLearningsSearch, generateLearningsLog } from './learnings';
|
||||
@@ -57,8 +58,8 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
NATIVE_LABEL: (ctx) => outsideVoiceFor(ctx).nativeLabel,
|
||||
OUTSIDE_PROVIDER: (ctx) => outsideVoiceFor(ctx).id,
|
||||
HOST_ID: (ctx) => ctx.host,
|
||||
OUTSIDE_PREFLIGHT: (ctx, args) => outsideVoicePreflight(ctx, { disabledBehavior: args?.[0] === 'opt-in' ? 'opt-in' : 'codex-only' }),
|
||||
OUTSIDE_INVOCATION: (ctx, args) => outsideVoiceInvocation(ctx, { timeoutMs: args?.[0] === 'spec' ? 120000 : 540000, gate: args?.[0] === 'spec' ? 'spec' : 'review', reasoningEffort: args?.[0] === 'spec' ? 'medium' : 'high' }),
|
||||
OUTSIDE_PREFLIGHT: (ctx, args) => outsideVoicePreflight(ctx, { disabledBehavior: args?.[0] === 'opt-in' ? 'opt-in' : 'codex-only', ...(args?.[0] === 'autoplan' || args?.[1] === 'plan-review' ? { role: 'plan-review' as const } : {}) }),
|
||||
OUTSIDE_INVOCATION: (ctx, args) => outsideVoiceInvocation(ctx, { timeoutMs: args?.[0] === 'spec' ? 120000 : 540000, gate: args?.[0] === 'spec' ? 'spec' : 'review', reasoningEffort: args?.[0] === 'spec' ? 'medium' : 'high', ...(args?.[0] === 'autoplan' || args?.[0] === 'spec' ? { role: 'plan-review' as const } : {}) }),
|
||||
OUTSIDE_PROVENANCE: (ctx, args) => outsideVoiceProvenance(ctx, args?.[0] ?? ctx.skillName),
|
||||
SLUG_EVAL: generateSlugEval,
|
||||
SLUG_SETUP: generateSlugSetup,
|
||||
@@ -125,6 +126,7 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
SCOPE_DRIFT: generateScopeDrift,
|
||||
DEPLOY_BOOTSTRAP: generateDeployBootstrap,
|
||||
CODEX_PLAN_REVIEW: generateCodexPlanReview,
|
||||
IMPLEMENTATION_MODEL_HANDOFF: generateImplementationModelHandoff,
|
||||
CODEX_DOC_REVIEW: generateCodexDocReview,
|
||||
PLAN_COMPLETION_AUDIT_SHIP: generatePlanCompletionAuditShip,
|
||||
PLAN_COMPLETION_GATE_SHIP: generatePlanCompletionGateShip,
|
||||
|
||||
@@ -416,7 +416,7 @@ completed output. Use private temporary paths, with no background jobs.` : `Run
|
||||
Finish a failed attempt's termination before fallback; consume only its completed
|
||||
output. No background jobs or shared temporary paths.`}
|
||||
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 300000 })}
|
||||
${outsideVoiceInvocation(ctx, { timeoutMs: 300000, role: 'plan-review' })}
|
||||
|
||||
Present the full output verbatim:
|
||||
|
||||
@@ -457,7 +457,7 @@ Immediately before dispatching, check the preflight result again. On
|
||||
do not dispatch. Otherwise, use this fallback for missing/broken CLI, failed
|
||||
authentication/model selection, a failed preflight${needsApprovalReadiness ? ' (including harness mismatch)' : ''}, or a failed outside invocation.
|
||||
The disabled branch never reaches this fallback.
|
||||
${needsApprovalReadiness ? '' : `On \`CODEX_MODE: ${outsideVoiceFor(ctx).id === 'codex' ? 'under_codex' : 'under_current_harness'}\`, report the setup repair and
|
||||
${needsApprovalReadiness ? '' : `On \`CODEX_MODE: under_current_harness\`, report the setup repair and
|
||||
\`outside_status: unavailable\`, run no outside CLI, and use the native subagent below.
|
||||
A native result never supplies outside coverage.`}`}`;
|
||||
}
|
||||
@@ -633,7 +633,7 @@ review. The user turns this off only by asking explicitly
|
||||
|
||||
**Preflight — decide whether and how the outside voice runs:**
|
||||
|
||||
${outsideVoicePreflight(ctx, { disabledBehavior: 'skip-all' })}
|
||||
${outsideVoicePreflight(ctx, { disabledBehavior: 'skip-all', role: 'plan-review' })}
|
||||
|
||||
${codexPlanOutcomeRouting(ctx, ceo, needsApprovalReadiness)}
|
||||
|
||||
|
||||
@@ -44,9 +44,12 @@ if { ${own}; }; then
|
||||
fi`;
|
||||
}
|
||||
|
||||
export function outsideVoicePreflight(ctx: TemplateContext, opts: { disabledBehavior: 'skip-all' | 'codex-only' | 'opt-in'; acceptedOnly?: boolean; nativeReview?: boolean }): string {
|
||||
/** Caller-owned workflow role. Only enumerated plan-review generators pass it; everything else stays no-role. */
|
||||
export type OutsideRole = 'plan-review';
|
||||
|
||||
export function outsideVoicePreflight(ctx: TemplateContext, opts: { disabledBehavior: 'skip-all' | 'codex-only' | 'opt-in'; acceptedOnly?: boolean; nativeReview?: boolean; role?: OutsideRole }): string {
|
||||
const v = outsideVoiceFor(ctx);
|
||||
if (v.id === 'codex' && opts.disabledBehavior !== 'opt-in') {
|
||||
if (v.id === 'codex' && opts.disabledBehavior !== 'opt-in' && !opts.role) {
|
||||
let preflight = outsideVoiceLabels(ctx, codexPreflight({ disabledBehavior: opts.disabledBehavior, nativeReview: opts.nativeReview }))
|
||||
.replace('```bash\n', `\`\`\`bash\n${runtimeRootPrelude(ctx)}\n`);
|
||||
if (['plan-eng-review', 'plan-ceo-review'].includes(ctx.skillName)) {
|
||||
@@ -81,7 +84,7 @@ if [ "$_OUTSIDE_CFG" = disabled ]; then
|
||||
${ctx.skillName === 'ship' && opts.disabledBehavior === 'opt-in' ? `Ship attempts this optional design check automatically when frontend review applies.
|
||||
The enabled value above carries that choice. No additional opt-in is needed.
|
||||
Step 11 keeps its separate outside-review switch.
|
||||
\`CODEX_MODE\` reports provider availability, not user consent; here the provider is **${v.label}**.` : `The historical \`CODEX_MODE\` variable describes **${v.label}** availability here.`} Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair ${v.label}; authentication failure: run \`${v.id === 'codex' ? 'codex login' : 'claude auth login'}\`. ${opts.disabledBehavior === 'skip-all' ? 'Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer.' : opts.disabledBehavior === 'codex-only' ? 'Disabled skips only the outside CLI; retain the native pass.' : ctx.skillName === 'ship' ? '' : 'Honor this caller’s existing opt-in/skip choice.'} ${opts.disabledBehavior === 'skip-all' ? 'Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled.' : opts.disabledBehavior === 'codex-only' ? 'Non-ready means missing outside coverage. Keep the required native pass without duplicating it.' : 'Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback.'} Never substitute another external provider.`;
|
||||
\`CODEX_MODE\` reports provider availability, not user consent; here the provider is **${v.label}**.` : `The historical \`CODEX_MODE\` variable describes **${v.label}** availability here.`} ${opts.role ? 'Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation.' : 'Authentication and configured model validity are checked by the actual invocation, without overriding either.'} Missing/broken CLI: install or repair ${v.label}; authentication failure: run \`${v.id === 'codex' ? 'codex login' : 'claude auth login'}\`. ${opts.disabledBehavior === 'skip-all' ? 'Disabled ends this entire extra review step, including the native fallback; record outside_status: disabled and continue after the section. Disabled is not an unavailable provider and never triggers a replacement reviewer.' : opts.disabledBehavior === 'codex-only' ? 'Disabled skips only the outside CLI; retain the native pass.' : ctx.skillName === 'ship' ? '' : 'Honor this caller’s existing opt-in/skip choice.'} ${opts.disabledBehavior === 'skip-all' ? 'Provider failure is missing outside coverage; follow the caller’s existing fallback only when reviews are enabled.' : opts.disabledBehavior === 'codex-only' ? 'Non-ready means missing outside coverage. Keep the required native pass without duplicating it.' : 'Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback.'} Never substitute another external provider.`;
|
||||
}
|
||||
|
||||
export interface OutsideCommandOptions {
|
||||
@@ -97,6 +100,8 @@ export interface OutsideCommandOptions {
|
||||
/** Creative proposals retain the recommendation gate with task-specific wording. */
|
||||
purpose?: 'design-direction';
|
||||
nativeAlreadyRequired?: boolean;
|
||||
/** Resolve the plan-review policy once in the invocation shell and bind it through probe and dispatch. */
|
||||
role?: OutsideRole;
|
||||
}
|
||||
|
||||
/** One self-contained shell body. No shell functions/variables survive between blocks. */
|
||||
@@ -108,15 +113,19 @@ export function outsideVoiceCommand(ctx: TemplateContext, opts: OutsideCommandOp
|
||||
const codex = opts.structuredBase
|
||||
? `codex review --base ${sh(opts.structuredBase)} -c "sandbox_mode=\\"\${_GSTACK_CODEX_SANDBOX:?}\\"" ${CODEX_REVIEW_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} < /dev/null`
|
||||
: `codex exec - -C "$_REPO_ROOT" -s "\${_GSTACK_CODEX_SANDBOX:?}" ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="${opts.reasoningEffort ?? 'high'}"' ${CODEX_WEB_SEARCH_FLAG} --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT"`;
|
||||
const kind = opts.structuredBase ? 'review' : 'exec';
|
||||
const codexSelection = opts.role
|
||||
? `source "${bin}/gstack-codex-probe" && _gstack_codex_role_ready ${kind} "$_REPO_ROOT" || exit $?`
|
||||
: `source "${bin}/gstack-codex-probe" && _gstack_codex_select_model ${kind} || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1`;
|
||||
const invocation = v.id === 'codex'
|
||||
? `source "${bin}/gstack-codex-probe" && _gstack_codex_select_model ${opts.structuredBase ? 'review' : 'exec'} || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
? `${codexSelection}
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper ${Math.ceil(opts.timeoutMs / 1000)} ${codex} >"$_OUTSIDE_TMP/${opts.structuredBase ? 'text' : 'events'}" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
${opts.structuredBase ? 'cat "$_OUTSIDE_TMP/text"' : 'cat "$_OUTSIDE_TMP/text" 2>/dev/null || tail -n 20 "$_OUTSIDE_TMP/events"'}`
|
||||
: `_OUTSIDE_EXIT=0
|
||||
"${bin}/gstack-claude-code" --cwd "$_REPO_ROOT" --access ${opts.access ?? 'none'} --timeout-ms ${opts.timeoutMs} <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/result.json" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
"${bin}/gstack-claude-code" --cwd "$_REPO_ROOT" --access ${opts.access ?? 'none'} --timeout-ms ${opts.timeoutMs}${opts.role ? ` --role ${opts.role}` : ''} <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/result.json" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
# Preserve session/usage/modelUsage from this JSON; multiple models have no invented primary.
|
||||
cat "$_OUTSIDE_TMP/result.json" || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; }
|
||||
if [ "$_OUTSIDE_EXIT" -eq 0 ]; then
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
/** Plan-approval handoff: recommend the runtime-resolved implementation model, never switch to it. */
|
||||
import { toShellPath, type TemplateContext } from './types';
|
||||
|
||||
export function generateImplementationModelHandoff(ctx: TemplateContext): string {
|
||||
const provider = ctx.host === 'claude' ? ' --provider anthropic' : ctx.host === 'codex' ? ' --provider openai' : '';
|
||||
return `**Implementation model:** relay the model and source from \`"${toShellPath(ctx.paths.binDir)}/gstack-models" resolve --role implementation${provider}\`${provider ? '' : ' (one per provider)'}. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.`;
|
||||
}
|
||||
@@ -228,12 +228,12 @@ ${setup ? 'Read `sections/browser-setup.md` in full unless already completed;\n'
|
||||
|
||||
export function generateQAReviewPreflight(ctx: TemplateContext): string {
|
||||
sectionPath(ctx, 'qa', 'exploratory');
|
||||
return `> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
|
||||
return `> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
|
||||
${ctx.skillName === 'review' ? 'Step 4 is read-only: defer charters, setup and probes to Step 4.7.\n' : ''}
|
||||
{{QA_RESOURCE:exploratory}}
|
||||
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
|
||||
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
|
||||
|
||||
Resolve QA's \`sections/...\` and \`templates/...\` paths from that installed QA SKILL.md directory, not the caller or product directory.`;
|
||||
QA's \`sections/...\` and \`templates/...\` paths resolve from installed QA SKILL.md, not the caller or product directory.`;
|
||||
}
|
||||
|
||||
export function generateQAReview(ctx: TemplateContext): string {
|
||||
|
||||
@@ -226,6 +226,7 @@ export const FREE_ONLY_PR_FILES = [
|
||||
'.github/workflows/dependency-review.yml',
|
||||
'.github/workflows/free-tests.yml',
|
||||
'.github/workflows/make-pdf-gate.yml',
|
||||
'.github/workflows/model-policy-freshness.yml',
|
||||
'.github/workflows/native-qualification.yml',
|
||||
'.github/workflows/osv-scanner.yml',
|
||||
'.github/workflows/platform-qualification.yml',
|
||||
|
||||
@@ -488,12 +488,12 @@ Never overwrite another run's reports. Batch only independent Reads.
|
||||
|
||||
**1. Load methods before any QA or explicit-verification probe.**
|
||||
|
||||
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below and await them. Templates cannot replace them.
|
||||
> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below in earlier responses. Never batch a probe (capture included) with its prerequisite Read; templates cannot replace Reads.
|
||||
|
||||
From the installed /ship SKILL.md's directory, Read `../qa/sections/exploratory.md` in full. If the caller directory is prefixed `gstack-ship`, use `../gstack-qa/sections/exploratory.md` instead. Use this host's installation, never the product tree. If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.
|
||||
Reading exploratory.md does not complete them: when it returns, Read the scope section and selected surface methods it lists, in order, and await them.
|
||||
After exploratory returns, Read scope then selected methods in order; it is only the entrypoint.
|
||||
|
||||
Resolve QA's `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory.
|
||||
QA's `sections/...` and `templates/...` paths resolve from installed QA SKILL.md, not the caller or product directory.
|
||||
|
||||
**2. List required checks.**
|
||||
Run the shared preflight; start its smoke guard once. Guard every smoke probe. For browsers, Read QA's `sections/browser-setup.md` for report-only rules.
|
||||
|
||||
@@ -869,6 +869,7 @@ Add to the standard template:
|
||||
`/autoplan` for the full review gauntlet).
|
||||
- **For implementation:** the issue itself is the handoff. The implementer can
|
||||
open it and execute without re-asking the user.
|
||||
**Implementation model:** relay the model and source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. A recommendation only; gstack cannot change this session. Switch nothing, start or spawn nothing, edit no config unless asked. On error relay its repair, not a model.
|
||||
- **`/ship` integration:** when `/ship` opens a PR for a worktree that contains
|
||||
a `/spec` archive (frontmatter `spec_issue_number: <N>`) AND the PR delivers
|
||||
the full spec (acceptance criteria checked off per `/ship`'s existing
|
||||
|
||||
@@ -474,6 +474,7 @@ Add to the standard template:
|
||||
`/autoplan` for the full review gauntlet).
|
||||
- **For implementation:** the issue itself is the handoff. The implementer can
|
||||
open it and execute without re-asking the user.
|
||||
{{IMPLEMENTATION_MODEL_HANDOFF}}
|
||||
- **`/ship` integration:** when `/ship` opens a PR for a worktree that contains
|
||||
a `/spec` archive (frontmatter `spec_issue_number: <N>`) AND the PR delivers
|
||||
the full spec (acceptance criteria checked off per `/ship`'s existing
|
||||
|
||||
@@ -149,7 +149,7 @@ else
|
||||
fi
|
||||
```
|
||||
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and configured model validity are checked by the actual invocation, without overriding either. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
|
||||
The historical `CODEX_MODE` variable describes **Codex** availability here. Authentication and the plan-review model (policy-selected, printed with its source) are checked by the actual invocation. Missing/broken CLI: install or repair Codex; authentication failure: run `codex login`. Honor this caller’s existing opt-in/skip choice. Any non-ready outcome is missing outside coverage; follow the caller’s existing fallback. Never substitute another external provider.
|
||||
|
||||
Write the prompt with the exact redaction-approved spec bytes using the Write tool; never shell-interpolate the raw draft. Keep hard delimiters and this boundary:
|
||||
|
||||
@@ -178,8 +178,7 @@ trap 'rm -rf "$_OUTSIDE_TMP"' EXIT
|
||||
_OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt"
|
||||
cat -- '<prepared-prompt-file>' >"$_OUTSIDE_INPUT" || exit 1
|
||||
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_select_model exec || exit 1
|
||||
_gstack_codex_sandbox_preflight >/dev/null || exit 1
|
||||
source "$HOME/.claude/skills/gstack/bin/gstack-codex-probe" && _gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?
|
||||
_gstack_codex_first_use_notice
|
||||
_OUTSIDE_EXIT=0
|
||||
_gstack_codex_timeout_wrapper 120 codex exec - -C "$_REPO_ROOT" -s "${_GSTACK_CODEX_SANDBOX:?}" -c "model=\"${_GSTACK_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="medium"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$?
|
||||
|
||||
@@ -60,7 +60,7 @@ dispatch. `spec-quality-gate-secret-sink.test.ts` enforces this.
|
||||
|
||||
**Dispatch (only when redaction passes):** No reviewer preflight/dispatch before the redaction decision. When blocked, STOP before Phase 5 and all downstream sinks. On --no-gate record skipped after redaction succeeds.
|
||||
|
||||
{{OUTSIDE_PREFLIGHT:opt-in}}
|
||||
{{OUTSIDE_PREFLIGHT:opt-in:plan-review}}
|
||||
|
||||
Write the prompt with the exact redaction-approved spec bytes using the Write tool; never shell-interpolate the raw draft. Keep hard delimiters and this boundary:
|
||||
|
||||
|
||||
@@ -2,7 +2,8 @@ import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { migrateClaudeCodeSkills } from '../lib/claude-code-migration';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { CLAUDE_CODE_RUNTIME_FILES as runtimeFiles, migrateClaudeCodeSkills } from '../lib/claude-code-migration';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const banner = '<!-- AUTO-GENERATED from SKILL.md.tmpl — do not edit directly -->\n<!-- Regenerate: bun run gen:skill-docs -->';
|
||||
@@ -15,7 +16,7 @@ function fixture() {
|
||||
const root = path.join(dir, 'checkout'); const home = path.join(dir, 'home');
|
||||
const codex = path.join(home, 'custom-codex', 'skills');
|
||||
const kiro = path.join(home, '.kiro', 'skills');
|
||||
for (const rel of ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts']) put(path.join(root, rel), rel);
|
||||
for (const rel of runtimeFiles) put(path.join(root, rel), rel);
|
||||
const oldRender = path.join(root, '.agents', 'skills', 'gstack-claude');
|
||||
put(path.join(oldRender, 'SKILL.md'), skill('gstack-claude'));
|
||||
fs.mkdirSync(codex, { recursive: true }); fs.mkdirSync(kiro, { recursive: true });
|
||||
@@ -40,6 +41,44 @@ function fixture() {
|
||||
}
|
||||
|
||||
describe('Claude wrapper installed-name migration', () => {
|
||||
test('copied policy tools and wrapper load without the source checkout', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
for (const rel of runtimeFiles) fs.copyFileSync(path.join(ROOT, rel), path.join(f.root, rel));
|
||||
put(path.join(f.codex, 'gstack-claude', 'SKILL.md'), skill('gstack-claude'));
|
||||
expect(f.run({ copy: true })).toEqual({ migrated: 1, pending: [] });
|
||||
fs.renameSync(f.root, `${f.root}-unavailable`);
|
||||
const runtime = path.join(f.codex, 'gstack');
|
||||
const env = {
|
||||
...process.env, HOME: f.home, USERPROFILE: f.home,
|
||||
GSTACK_STATE_ROOT: path.join(f.home, 'state'), CODEX_HOME: path.dirname(f.codex),
|
||||
CLAUDE_CONFIG_DIR: path.join(f.home, '.claude'),
|
||||
GSTACK_CLAUDE_MODEL: '', GSTACK_CODEX_MODEL: '', ANTHROPIC_BASE_URL: '', OPENAI_BASE_URL: '',
|
||||
CLAUDE_CODE_USE_BEDROCK: '', CLAUDE_CODE_USE_VERTEX: '', CLAUDE_CODE_USE_FOUNDRY: '',
|
||||
BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0',
|
||||
};
|
||||
const set = spawnSync('bash', [path.join(runtime, 'bin/gstack-config'), 'set', 'plan_review_tier', 'smart'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(set.status).toBe(0);
|
||||
const inspect = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-models'), 'resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(inspect.status).toBe(0);
|
||||
expect(JSON.parse(inspect.stdout).selections[0].requestedModel).toBe('claude-opus-5-5');
|
||||
const wrapper = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-claude-code'), '--help'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(wrapper.status).toBe(1);
|
||||
expect(JSON.parse(wrapper.stdout).error.code).toBe('arguments');
|
||||
const gate = spawnSync(process.execPath, ['-e', 'const {classifyOutsideReview} = await import(process.argv[1]); console.log(classifyOutsideReview({text:"",gate:"review",exit:1}).verdict)', path.join(runtime, 'lib/outside-review-result.ts')], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(gate.status).toBe(0);
|
||||
expect(gate.stdout.trim()).toBe('unavailable');
|
||||
} finally { fs.rmSync(f.dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('migrates existing Codex and Kiro installs independently of the selected setup host', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
|
||||
@@ -937,15 +937,17 @@ describe('codex broken-install detection (#2742)', () => {
|
||||
// hand-maintained copy that isn't resolver-generated — it must capture the
|
||||
// probe's exit code and route 2 to its own broken-install arm, or /autoplan
|
||||
// prints the wrong remedy for a broken binary.
|
||||
test('autoplan preflight (tmpl + rendered) captures the probe exit and routes 2 to broken-install', () => {
|
||||
for (const rel of ['autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md']) {
|
||||
const raw = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
const src = rel.endsWith('.tmpl') ? raw.replace('{{OUTSIDE_PREFLIGHT:autoplan}}', RESOLVERS.OUTSIDE_PREFLIGHT({ host: 'claude', paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: rel }, ['autoplan'])) : raw;
|
||||
expect(src).toContain('_gstack_codex_model_probe; _CODEX_MP=$?');
|
||||
expect(src).toMatch(/_CODEX_MP" -eq 2/);
|
||||
expect(src).toContain('binary cannot run');
|
||||
expect(src).not.toContain('elif ! _gstack_codex_model_probe');
|
||||
test('autoplan (tmpl + rendered): availability pays for no probe; the invocation probe exit, 2 included, ends it', () => {
|
||||
const ctx = { host: 'claude' as const, paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: 'autoplan/SKILL.md.tmpl' };
|
||||
const rendered = ['autoplan/SKILL.md', ...['ceo', 'design', 'dx', 'eng'].map(phase => `autoplan/sections/${phase}-phase.md`)]
|
||||
.map(rel => fs.readFileSync(path.join(ROOT, rel), 'utf-8')).join('\n');
|
||||
for (const src of [RESOLVERS.OUTSIDE_PREFLIGHT(ctx, ['autoplan']) + RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan']), rendered]) {
|
||||
expect(src).toContain('_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?');
|
||||
expect(src).not.toMatch(/_gstack_codex_model_probe|_CODEX_MP/);
|
||||
}
|
||||
const role = fs.readFileSync(PROBE, 'utf-8').match(/_gstack_codex_role_ready\(\) \{[\s\S]*?\n\}/)![0];
|
||||
expect(role).toContain('_gstack_codex_model_probe "$1"');
|
||||
expect(role.trim().split('\n').at(-2)!.trim()).toBe('_gstack_codex_model_probe "$1"');
|
||||
});
|
||||
|
||||
test('the preflight resolver routes exit 2 to broken_install', () => {
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* Bounded multi-signature Codex probe cache (stubbed codex, no API spend):
|
||||
* 16 entries, atomic publication, same-signature serialization with a
|
||||
* bounded lock, abandoned-owner recovery, and unverified (never a duplicate
|
||||
* paid probe) on lock timeout or unreadable cache.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
const STUB = `#!/usr/bin/env bash
|
||||
[ "$1" = sandbox ] && exit 0
|
||||
printf '%s\\n' "$*" >> "$STUB_LOG"
|
||||
[ -n "\${STUB_SLEEP:-}" ] && sleep "$STUB_SLEEP"
|
||||
case "\${STUB_MODE:-ok}" in
|
||||
ok) echo OK; exit 0 ;;
|
||||
model400) echo 'ERROR: {"type":"error","status":400,"error":{"message":"The model is not supported when using Codex with a ChatGPT account."}}' >&2; exit 1 ;;
|
||||
quota) echo "ERROR: You've hit your usage limit. Try again at 3 AM." >&2; exit 1 ;;
|
||||
esac
|
||||
`;
|
||||
|
||||
interface Fixture { home: string; codexHome: string; state: string; log: string; cache: string; locks: string; env: Record<string, string> }
|
||||
|
||||
function fixture(): Fixture {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-probe-cache-'));
|
||||
const bin = path.join(home, 'bin');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const state = path.join(home, 'state');
|
||||
for (const dir of [bin, codexHome, state]) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(bin, 'codex'), STUB, { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n');
|
||||
fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}');
|
||||
const log = path.join(home, 'stub.log');
|
||||
return {
|
||||
home, codexHome, state, log,
|
||||
cache: path.join(state, '.codex-model-probe'),
|
||||
locks: path.join(state, '.codex-model-probe.locks'),
|
||||
env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: home, CODEX_HOME: codexHome, GSTACK_HOME: state, STUB_LOG: log, _TEL: 'off' },
|
||||
};
|
||||
}
|
||||
|
||||
function probe(f: Fixture, env: Record<string, string> = {}, call = '_gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"') {
|
||||
const r = spawnSync('bash', ['-c', `source "${PROBE}"\n${call}`], { env: { ...f.env, ...env }, encoding: 'utf8', timeout: 20000 });
|
||||
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status };
|
||||
}
|
||||
|
||||
function probeAsync(f: Fixture, env: Record<string, string>): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('bash', ['-c', `source "${PROBE}"\n_gstack_codex_model_probe; echo "rc=$?"`], { env: { ...f.env, ...env } });
|
||||
let out = '';
|
||||
child.stdout.on('data', chunk => { out += chunk; });
|
||||
const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('probe hung')); }, 30000);
|
||||
child.on('close', () => { clearTimeout(timer); resolve(out); });
|
||||
});
|
||||
}
|
||||
|
||||
const calls = (f: Fixture) => (fs.existsSync(f.log) ? fs.readFileSync(f.log, 'utf8').split('\n').filter(Boolean).length : 0);
|
||||
const entries = (f: Fixture) => fs.readFileSync(f.cache, 'utf8').split('\n').filter(line => /^MODEL_\S+ \d+ \d+$/.test(line));
|
||||
const model = (name: string) => ({ GSTACK_CODEX_MODEL: name });
|
||||
const withFixture = (body: (f: Fixture) => void | Promise<void>) => async () => {
|
||||
const f = fixture();
|
||||
try { await body(f); } finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
};
|
||||
|
||||
describe('bounded multi-signature probe cache', () => {
|
||||
test('A -> B -> A makes exactly two paid probes', withFixture((f) => {
|
||||
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK');
|
||||
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK');
|
||||
const again = probe(f, model('gpt-a'));
|
||||
expect(again.stdout).toContain('MODEL_OK (cached)');
|
||||
expect(calls(f)).toBe(2);
|
||||
expect(entries(f)).toHaveLength(2);
|
||||
expect(fs.existsSync(f.locks) ? fs.readdirSync(f.locks) : []).toEqual([]);
|
||||
}));
|
||||
|
||||
test('concurrent same-signature misses share one probe', withFixture(async (f) => {
|
||||
const outputs = await Promise.all([0, 1, 2, 3].map(() => probeAsync(f, { ...model('gpt-same'), STUB_SLEEP: '1' })));
|
||||
expect(calls(f)).toBe(1);
|
||||
for (const out of outputs) expect(out).toMatch(/MODEL_OK(?: \(cached\))?\nrc=0/);
|
||||
expect(outputs.filter(out => out.includes('(cached)'))).toHaveLength(3);
|
||||
expect(entries(f)).toHaveLength(1);
|
||||
}));
|
||||
|
||||
test('the 17th signature evicts the oldest entry; the file never exceeds 16', withFixture((f) => {
|
||||
for (let i = 1; i <= 17; i++) expect(probe(f, model(`gpt-m${i}`)).stdout).toContain('MODEL_OK');
|
||||
expect(entries(f)).toHaveLength(16);
|
||||
expect(probe(f, model('gpt-m17')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(probe(f, model('gpt-m2')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(calls(f)).toBe(17);
|
||||
expect(probe(f, model('gpt-m1')).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(18);
|
||||
expect(entries(f)).toHaveLength(16);
|
||||
}));
|
||||
|
||||
test('expired entries are pruned and never hit; TTLs and negative classifications are unchanged', withFixture((f) => {
|
||||
probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
|
||||
probe(f, model('gpt-good'));
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const lines = fs.readFileSync(f.cache, 'utf8').split('\n').filter(Boolean)
|
||||
.map(line => line.replace(/^(MODEL_UNUSABLE) (\d+)/, (_, s) => `${s} ${now - 901}`));
|
||||
fs.writeFileSync(f.cache, `${lines.join('\n')}\n`);
|
||||
expect(probe(f, { ...model('gpt-good') }).stdout).toContain('MODEL_OK (cached)');
|
||||
const bad = probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
|
||||
expect(bad.stdout).toContain('MODEL_UNUSABLE\n');
|
||||
expect(bad.stdout).toContain('rc=1');
|
||||
expect(calls(f)).toBe(3);
|
||||
expect(probe(f, { ...model('gpt-bad'), STUB_MODE: 'ok' }).stdout).toContain('MODEL_UNUSABLE (cached)');
|
||||
expect(calls(f)).toBe(3);
|
||||
const future = fs.readFileSync(f.cache, 'utf8').replace(/^MODEL_OK \d+/m, `MODEL_OK ${now + 999}`);
|
||||
fs.writeFileSync(f.cache, future);
|
||||
expect(probe(f, model('gpt-good')).stdout).not.toContain('(cached)');
|
||||
}));
|
||||
|
||||
test('quota detail lines survive among other entries and are relayed on a hit', withFixture((f) => {
|
||||
probe(f, model('gpt-x'));
|
||||
expect(probe(f, { ...model('gpt-q'), STUB_MODE: 'quota' }).stdout).toContain('rc=4');
|
||||
probe(f, model('gpt-y'));
|
||||
const hit = probe(f, model('gpt-q'));
|
||||
expect(hit.stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)');
|
||||
expect(hit.stdout).toContain("ERROR: You've hit your usage limit. Try again at 3 AM.");
|
||||
expect(hit.stdout).toContain('rc=4');
|
||||
expect(probe(f, { ...model('gpt-q'), GSTACK_CODEX_PROBE_RETRY: '1' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(4);
|
||||
expect(entries(f)).toHaveLength(3);
|
||||
}));
|
||||
|
||||
test('a pre-v2 one-entry cache migrates by TTL: fresh honored, expired re-probed, stray lines ignored', withFixture((f) => {
|
||||
probe(f);
|
||||
const [status, ts, sig] = entries(f)[0].split(' ');
|
||||
expect(status).toBe('MODEL_OK');
|
||||
fs.writeFileSync(f.cache, `MODEL_OK ${ts} ${sig}\nan unprefixed legacy line\n`);
|
||||
expect(probe(f).stdout).toContain('MODEL_OK (cached)');
|
||||
fs.writeFileSync(f.cache, `MODEL_OK ${Number(ts) - 3700} ${sig}\n`);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(2);
|
||||
}));
|
||||
|
||||
test('an interrupted writer leaves a truncated line and temp file that never count as a hit', withFixture((f) => {
|
||||
probe(f, model('gpt-a'));
|
||||
fs.appendFileSync(f.cache, 'MODEL_OK 17');
|
||||
fs.writeFileSync(`${f.cache}.tmp.99999`, 'MODEL_OK 1 1\n');
|
||||
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK\n');
|
||||
expect(fs.readFileSync(f.cache, 'utf8')).not.toContain('MODEL_OK 17\n');
|
||||
expect(entries(f)).toHaveLength(2);
|
||||
}));
|
||||
|
||||
test('config, auth and binary changes still invalidate a cached entry', withFixture((f) => {
|
||||
probe(f);
|
||||
const future = Date.now() / 1000 + 10;
|
||||
fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
fs.utimesSync(path.join(f.codexHome, 'auth.json'), future + 5, future + 5);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
expect(probe(f, { CODEX_API_KEY: 'k-test' }).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(4);
|
||||
}));
|
||||
});
|
||||
|
||||
describe('probe lock recovery and failure', () => {
|
||||
const sigOf = (f: Fixture) => {
|
||||
probe(f);
|
||||
const sig = entries(f)[0].split(' ')[2];
|
||||
fs.rmSync(f.cache);
|
||||
fs.rmSync(f.log);
|
||||
return sig;
|
||||
};
|
||||
|
||||
test('a dead owner on this host is recovered at once, without a duplicate probe', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
const dead = spawnSync('bash', ['-c', 'echo $$'], { timeout: 5000 }).stdout.toString().trim();
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${dead} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`);
|
||||
const r = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '3' });
|
||||
expect(r.stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
expect(fs.existsSync(lock)).toBe(false);
|
||||
}));
|
||||
|
||||
test('a live owner past the stale bound is abandoned; within it the waiter is unverified without probing', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now}\n`);
|
||||
const busy = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' });
|
||||
expect(busy.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (another probe of this model still holds its lock)');
|
||||
expect(busy.stdout).toContain('rc=0 state=inconclusive');
|
||||
expect(calls(f)).toBe(0);
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now - 120}\n`);
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('a lock without its owner file is the creation window until it is 5 seconds old', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' }).stdout).toContain('MODEL_PROBE_INCONCLUSIVE');
|
||||
const old = Date.now() / 1000 - 30;
|
||||
fs.utimesSync(lock, old, old);
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('an unreadable cache or uncreatable lock is unverified and launches no paid probe', withFixture((f) => {
|
||||
fs.mkdirSync(f.cache);
|
||||
const dir = probe(f);
|
||||
expect(dir.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
|
||||
expect(dir.stdout).toContain('rc=0 state=inconclusive');
|
||||
fs.rmdirSync(f.cache);
|
||||
fs.writeFileSync(f.cache, '');
|
||||
fs.chmodSync(f.cache, 0o000);
|
||||
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
|
||||
fs.chmodSync(f.cache, 0o644);
|
||||
fs.rmSync(f.cache);
|
||||
fs.writeFileSync(f.locks, 'not a directory');
|
||||
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
|
||||
expect(calls(f)).toBe(0);
|
||||
}));
|
||||
|
||||
test('an owner record that cannot be written is lock I/O failure: unverified, no paid probe, no stray lock', withFixture((f) => {
|
||||
fs.mkdirSync(f.locks, { recursive: true });
|
||||
const r = probe(f, {}, '_gstack_codex_select_model exec 2>/dev/null; umask 0777; _gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"');
|
||||
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot record ownership of the probe lock');
|
||||
expect(r.stdout).toContain('rc=0 state=inconclusive');
|
||||
expect(calls(f)).toBe(0);
|
||||
expect(fs.readdirSync(f.locks)).toEqual([]);
|
||||
expect(fs.existsSync(f.cache)).toBe(false);
|
||||
expect(probe(f).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('a cache write failure keeps the real probe result', withFixture((f) => {
|
||||
fs.chmodSync(f.state, 0o500);
|
||||
try {
|
||||
const r = probe(f, { STUB_MODE: 'model400' });
|
||||
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
|
||||
fs.chmodSync(f.state, 0o700);
|
||||
fs.mkdirSync(f.locks);
|
||||
fs.writeFileSync(path.join(f.locks, 'write'), 'blocks the write lock');
|
||||
const real = probe(f, { STUB_MODE: 'model400' });
|
||||
expect(real.stdout).toContain('MODEL_UNUSABLE\n');
|
||||
expect(real.stdout).toContain('rc=1');
|
||||
expect(fs.existsSync(f.cache)).toBe(false);
|
||||
} finally { fs.chmodSync(f.state, 0o700); }
|
||||
}));
|
||||
|
||||
test('sourcing the probe still leaves the caller shell untouched', () => {
|
||||
const r = spawnSync('bash', ['-c', `B=$(set -o; trap -p; echo "$IFS"); source "${PROBE}"; A=$(set -o; trap -p; echo "$IFS"); [ "$A" = "$B" ] && echo CLEAN`], { encoding: 'utf8', timeout: 5000 });
|
||||
expect(r.stdout.trim()).toBe('CLEAN');
|
||||
});
|
||||
});
|
||||
@@ -90,7 +90,7 @@ describe('under-codex wiring renders (#2519)', () => {
|
||||
test('all three codexPreflight consumers render the probe', () => {
|
||||
for (const file of [
|
||||
path.join(ROOT, 'ship', 'sections', 'adversarial.md'),
|
||||
path.join(ROOT, 'plan-ceo-review', 'sections', 'review-sections.md'),
|
||||
path.join(ROOT, 'review', 'sections', 'adversarial.md'),
|
||||
path.join(ROOT, 'document-release', 'sections', 'release-body.md'),
|
||||
]) {
|
||||
const rendered = fs.readFileSync(file, 'utf-8');
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { FRESHNESS_SOURCES, parseOfficialSource, sha256, type SourceId, type SourceResult } from '../../lib/model-policy-freshness-sources';
|
||||
import type { CatalogIdentity, FreshnessObservation } from '../../lib/model-policy-freshness';
|
||||
|
||||
export const freshnessCatalog: CatalogIdentity = {
|
||||
sha256: sha256('catalog-v1'), sourceSha256: sha256('catalog-source-v1'), version: 1,
|
||||
models: [
|
||||
{ provider: 'anthropic', tier: 'frontier', modelId: 'claude-fable-5-1' },
|
||||
{ provider: 'anthropic', tier: 'smart', modelId: 'claude-opus-5-5' },
|
||||
{ provider: 'openai', tier: 'frontier', modelId: 'gpt-6-astra' },
|
||||
{ provider: 'openai', tier: 'smart', modelId: 'gpt-6.1-sol' },
|
||||
],
|
||||
};
|
||||
|
||||
export function freshnessFixture(id: SourceId): string {
|
||||
return readFileSync(new URL(`./model-policy-freshness/${id}.md`, import.meta.url), 'utf8');
|
||||
}
|
||||
|
||||
export function freshnessObservation(id = '100', at = '2026-10-07T12:00:00.000Z', overrides: Partial<Record<SourceId, string | null>> = {}): FreshnessObservation {
|
||||
const sources: SourceResult[] = FRESHNESS_SOURCES.map(source => {
|
||||
const text = overrides[source.id] === undefined ? freshnessFixture(source.id) : overrides[source.id];
|
||||
return text === null ? { id: source.id, error: 'http-503' } : {
|
||||
id: source.id, receipt: { id: source.id, url: source.url, finalUrl: source.url, sha256: sha256(text!), bytes: Buffer.byteLength(text!), parsed: parseOfficialSource(source.id, text!) },
|
||||
};
|
||||
});
|
||||
return { catalog: structuredClone(freshnessCatalog), checkedAt: at, run: { id, attempt: 1, startedAt: new Date(Date.parse(at) - 1000).toISOString(), headSha: 'a'.repeat(40) }, sources };
|
||||
}
|
||||
|
||||
export function retirementFixture(): string {
|
||||
return freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027', 'claude-fable-5-1 | Deprecated | October 1, 2026 | November 1, 2026');
|
||||
}
|
||||
|
||||
export function freshnessReplacementObservation(): FreshnessObservation {
|
||||
const observation = freshnessObservation('101', '2026-10-08T12:00:00.000Z', {
|
||||
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
});
|
||||
observation.catalog.models[0].modelId = 'claude-fable-5-2';
|
||||
observation.catalog.sha256 = 'b'.repeat(64);
|
||||
observation.catalog.sourceSha256 = 'c'.repeat(64);
|
||||
return observation;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
# Model deprecations
|
||||
|
||||
Partner-operated platforms (Amazon Bedrock and Google Cloud) set their own retirement schedules.
|
||||
|
||||
## Model status
|
||||
|
||||
| API model name | Current state | Deprecated | Tentative retirement date |
|
||||
| --- | --- | --- | --- |
|
||||
| claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027 |
|
||||
| claude-opus-5-5 | Active | N/A | Not sooner than September 22, 2027 |
|
||||
| claude-mythos-preview | Deprecated | June 9, 2026 | To be announced |
|
||||
| claude-opus-4-1-20250805 | Retired | June 5, 2026 | August 5, 2026 |
|
||||
|
||||
## Deprecation history
|
||||
|
||||
Amazon Bedrock partner discussion mentions `claude-fable-5-1` retirement; this is not a public API lifecycle row.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Models overview
|
||||
|
||||
## Compare models
|
||||
|
||||
If you're unsure which model to use, start with [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) for most workloads. Use [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) for demanding reasoning and long-horizon agentic work.
|
||||
|
||||
| Feature | Claude Fable 5.1 | Claude Opus 5.5 | Claude Sonnet 5.5 | Claude Haiku 5.5 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Model page | [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) | [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) | [Claude Sonnet 5.5](https://platform.claude.com/docs/en/models/sonnet-5-5/overview) | [Claude Haiku 5.5](https://platform.claude.com/docs/en/models/haiku-5-5/overview) |
|
||||
| Claude API ID | `claude-fable-5-1` | `claude-opus-5-5` | `claude-sonnet-5-5` | `claude-haiku-5-5` |
|
||||
| Amazon Bedrock ID | `anthropic.claude-fable-5-1` | `anthropic.claude-opus-5-5` | `anthropic.claude-sonnet-5-5` | `anthropic.claude-haiku-5-5` |
|
||||
|
||||
* **Pricing:** prompt cache reads cost 10% of the base input price (2.5% on Claude Fable 5.1 and Claude Mythos 5.1, 5% on Claude Opus 5.5). See Pricing for per-platform pricing. Incidental ID: `claude-mythos-5-1`.
|
||||
|
||||
## Using the Models API
|
||||
|
||||
Legacy `claude-fable-5` and unrelated `claude-mythos-5-1` are not recommendations.
|
||||
@@ -0,0 +1,24 @@
|
||||
# Deprecations
|
||||
|
||||
This page lists all API deprecations, along with recommended replacements.
|
||||
|
||||
## Upcoming deprecations
|
||||
|
||||
### 2026-10-01: GPT-5.3-Codex
|
||||
|
||||
| Shutdown date | Model / system | Recommended replacement |
|
||||
| --- | --- | --- |
|
||||
| Apr 1, 2027 | `gpt-5.3-codex` | `gpt-6-sol` |
|
||||
|
||||
## Past deprecations
|
||||
|
||||
| Shutdown date | Model snapshot | Substitute model |
|
||||
| --- | --- | --- |
|
||||
| Aug 5, 2026 | `gpt-old` | `gpt-6-astra` |
|
||||
| 2026‑03‑26 | `gpt-unicode-date` | `gpt-6-astra` |
|
||||
| 2026-02-12 | `gpt-iso-date` | `gpt-6-astra` |
|
||||
| at earliest 2024-06-13 | `gpt-date-floor` | `gpt-6-astra` |
|
||||
| 2026-09-24 | Videos API | --- |
|
||||
| 2024-01-04 | `/v1/edits` | `/v1/chat/completions` |
|
||||
|
||||
Incidental replacement text `gpt-6-astra` is not a deprecated model row.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Models
|
||||
|
||||
## Featured models
|
||||
|
||||
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Start here for complex reasoning and coding.
|
||||
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Balance intelligence and cost.
|
||||
- [GPT-6 Luna](/api/docs/models/gpt-6-luna.md): Optimize cost-sensitive, high-volume workloads.
|
||||
|
||||
## Browse our full catalog of models
|
||||
|
||||
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Our most capable model for the most demanding work.
|
||||
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Near-Astra performance for complex work at a lower cost.
|
||||
- [GPT-9 Preview](/api/docs/models/gpt-9-preview.md): Incidental full-catalog listing, not a featured recommendation.
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"source": "ship-exploratory-late-input, mtb1007a, 2026-10-07; public synthetic fixture events only",
|
||||
"events": [
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/exploratory.md"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
|
||||
"type": "tool_result",
|
||||
"content": "1\t<!-- AUTO-GENERATED from exploratory.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Shared exploratory QA\n4\t\n5\tThe **caller** (/qa, /qa-only, /review or /ship) owns decisions, tests, fixes and publication. Discovery writes only reports/evidence\n6\tand owned fixture state; no workflows, framework installs or publication.\n7\t\n8\tComplete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation.\n9\t1. Read `sections/scope.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and select the surfaces.\n10\t2. Read the selected surface methods below in full.\n11\t\n12\t**Functional surfaces:**\n13\tRead `sections/system-functional.md` in full.\n14\t\n15\t**Browser surfaces only:**\n16\tRead `sections/qa-patterns.md` in full.\n17\t\n18\tMissing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks. Report QA setup blockers.\n19\t\n20\t## 1. Charter and preflight\n21\t\n22\tReuse resolved REPORT_DIR; otherwise own a fresh `.gstack/qa-reports` subdirectory.\n23\tWrite a **charter** per behavior: contract, risk, entrypoint, isolation, exit condition, source, commands and inputs. Save charters as Markdown in the report.\n24\t\n25\tFor /review and /ship, no plan/server is required.\n26\tStop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces).\n27\tExplicit plan checks and revalidation remain required beyond this smoke budget.\n28\tFor /qa and /qa-only:\n29\t- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900.\n30\t- Functional Full, Quick and Regression have no default total timer.\n31\tSet SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.\n32\tWithout a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.\n33\tStop when scoped contracts are tested or blocked.\n34\tClocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.\n35\tNames used below (quote every path):\n36\t- PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run.\n37\t- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.\n38\t- DEADLINE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-deadline`, the deadline guard: `start` creates DEADLINE_FILE, `status` prints `remainingMs` and `expired`, and `run` executes one command, stopping it at the deadline.\n39\t- EVIDENCE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence`, the functional evidence recorder: `capture` runs one command, stores its exit code, stdout and stderr under `PROBE_DIR/.qa-evidence/NNN/`, and prints status `complete`, `incomplete` or `sensitive`.\n40\t\n41\tStart once before baseline: `bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]` if bounded.\n42\tEARLIER_UTC = caller's absolute deadline, if set.\n43\tFunctional: `bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS`.\n44\tUnbounded: use `--timeout-ms MS` instead of `--deadline DEADLINE_FILE`.\n45\t--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of `PROBE_DIR/.qa-evidence/NNN/observation.json` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.\n46\tBounded browsers: `bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS`. No detached probes.\n47\tNever reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.\n48\t\n49\t## 2. Probe loop\n50\t\n51\tEach probe is one native command/interaction plus checks, excluding bookkeeping.\n52\tNever batch probes.\n53\t\n54\t1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.\n55\t2. **Decide whether another probe is needed.** If bounded, run `bun DEADLINE_TOOL status DEADLINE_FILE`.\n56\t If expired or no safe next probe remains, STOP exploration; write the report (\u00a74), not a checkpoint.\n57\t **Publish before probing.** Create (browser) or compose (functional) `exploration-NNN.json` in the probe directory, beside its deadline if bounded, with exactly four top-level fields:\n58\t observationCommand: last completed probe's full outer command, including guard.\n59\t observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text.\n60\t hypothesis: why nextCommand. nextCommand: exact command/request, guarded if bounded.\n61\t PrLine truncated
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "bun /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence capture /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports 001 --public --deadline /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports/deadline.json -- bun scripts/probe.ts 3",
|
||||
"description": "Capture happy-path probe for input 3"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
|
||||
"type": "tool_result",
|
||||
"content": "{\"id\":\"probe-ba7e52d4-ad96-4df9-97d0-6dd79e52f184\",\"charter\":\"happy\",\"input\":\"3\",\"snapshot\":\"936ff5b3b13172fb2357a120804c1cf5cac68d911c82cae5bce630f04dd62b57\",\"status\":\"pass\",\"stdout\":\"6\\n\",\"stderr\":\"\",\"exit\":0}\n\nQA_EVIDENCE {\"producer\":\"gstack-qa-evidence\",\"version\":1,\"action\":\"capture\",\"id\":\"001\",\"status\":\"complete\",\"sha256\":\"135ce1a8f804e3c6d953e0f284ab4e1e7fe5ded56f3a35706ea73f630876d0be\",\"exitCode\":0,\"signal\":null,\"publicOutput\":true,\"startedAt\":\"2026-10-07T22:13:30.620Z\",\"completedAt\":\"2026-10-07T22:13:30.675Z\",\"durationMs\":55,\"remainingMs\":182686,\"next\":\"Another probe requires a checkpoint anchored on capture 001: add --after 001 --hypothesis 'TEXT' before --. To stop exploring, run none.\"}",
|
||||
"is_error": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool_use_result": {
|
||||
"interrupted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeRZyA2j8av6NchLTmY",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/system-functional.md"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
|
||||
"type": "tool_result",
|
||||
"content": "1\t<!-- AUTO-GENERATED from system-functional.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Functional QA with repository-native tools\n4\t\n5\tUse documented repository commands, CLI/API clients and job/queue tools, not a new\n6\tharness or browser substitution.\n7\t\n8\t## Functional modes\n9\t\n10\tFor /qa and /qa-only, within the selected scope:\n11\t- **Full** (default): cover every applicable documented contract below.\n12\t- **Quick** (`--quick`): check success and the highest-risk changed edge; mark other\n13\t contracts not run.\n14\t- **Regression** (`--regression <previous-report>`): before probes, read the supplied\n15\t functional report and linked replay evidence. A missing, unreadable or wrong-target\n16\t baseline blocks regression mode. A browser-only `baseline.json` is not a functional\n17\t baseline. Re-establish owned setup; replay prior failed probes against the documented\n18\t expectation, never recorded buggy output, then check changed adjacent behavior.\n19\t Preserve the prior report; report fixed, still failing and new findings separately.\n20\t Missing safe replay inputs block affected probes, never count as passes.\n21\t\n22\tMixed runs apply each surface's mode separately. /review and /ship retain their caller's\n23\tbounded smoke and explicit plan checks, not Full exploration.\n24\t\n25\t## Contract map\n26\t\n27\tRecord each contract/source, isolated setup, exact probe, expectation and outcome:\n28\tpass/fail/blocked/not run/inconclusive/not applicable (reason).\n29\t\n30\t| Contract | Observe |\n31\t|---|---|\n32\t| Successful execution | Expected return/output and final business effect, not just launch/acceptance |\n33\t| Invalid/missing input | Declared rejection, correct status and no forbidden state change |\n34\t| Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary |\n35\t| CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes |\n36\t| State transitions | Initial, intermediate and completed/failed states and their permitted transitions |\n37\t| Timeout/cancellation | Deadline, partial state, termination of owned work and recovery |\n38\t| Retry | Attempts/backoff/terminal state promised by the repository; no unbounded retry |\n39\t| Duplicates/idempotency | Repeated request/event and number of durable effects under the documented guarantee |\n40\t| Concurrency/order | Controlled competing operations in both relevant completion orders; final invariant |\n41\t| Partial-failure recovery | Interrupt after an effect, restart/replay, inspect completion/dead-letter state and duplicates |\n42\t\n43\tDo not impose universal exactly-once delivery. Separate acceptance, enqueue, processing,\n44\tretry/dead-letter and final effect; 2xx is not completion. Expected rejection/injected\n45\tfailure may pass; a missing service preventing execution blocks coverage.\n46\t\n47\t## Execute and retain evidence\n48\t\n49\t1. Apply the shared isolation/permission preflight. Verify cwd, command, environment\n50\t NAMES and safe reset; use synthetic data/credentials.\n51\t2. Follow the shared exploratory loop's order and written checkpoints.\n52\t For every probe, inspect initial/final durable state and retain exit/status and\n53\t stdout/stderr separately without masking failure.\n54\t3. On timeout, retain partial output/state and stop only owned work. Record setup errors\n55\t and untested contracts; never patch product code to hide missing prerequisites.\n56\t4. Record exact command or method/path/headers/body, setup/reset, expected contract/source,\n57\t observed output/state, revision/runtime, evidence paths and limits. Secrets are referenced\n58\t only by environment name. Disclose replay limits caused by redaction.\n59\t5. Use `templates/functional-report-template.md` relative to the installed QA SKILL.md.\n60\t Preserve evidence before owned cleanup and disclose leftovers. Return to the caller\n61\t without expanding discovery authority.\n62\t"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -123,6 +123,8 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'],
|
||||
'test/fixtures/module-size-ratchet.json': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
|
||||
'test/fixtures/module-size/**': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
|
||||
'test/fixtures/model-policy-freshness.ts': ['test/model-policy-freshness.test.ts', 'test/model-policy-workflow.test.ts'],
|
||||
'test/fixtures/model-policy-freshness/**': ['test/fixtures/model-policy-freshness.ts'],
|
||||
'test/fixtures/native-auq-boxed-full-body-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
@@ -138,6 +140,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'],
|
||||
'test/fixtures/plan-floor-edit-wrap-37176835584.json': ['test/plan-floor-edit-wrap.test.ts'],
|
||||
'test/fixtures/plan-format-kind-notes.json': ['test/plan-format-kind-note.test.ts'],
|
||||
'test/fixtures/qa-caller-read-probe-mtb1007a.json': ['test/qa-exploratory-callers.test.ts'],
|
||||
'test/fixtures/plans/autoplan-dashboard.md': ['test/autoplan-init.test.ts', 'test/autoplan-preconfigured-onboarding-ar.test.ts', 'test/autoplan-snapshot.test.ts'],
|
||||
'test/fixtures/plans/autoplan-focus-appearance-design.md': [],
|
||||
'test/fixtures/plans/autoplan-focus-appearance.md': [],
|
||||
|
||||
@@ -110,6 +110,11 @@ export function callerTools(transcript: unknown[]): CallerTool[] {
|
||||
return tools;
|
||||
}
|
||||
|
||||
export function callerProbeReadErrors(probe: CallerTool, resources: readonly (readonly [string, CallerTool | undefined])[]): string[] {
|
||||
const premature = (read: CallerTool) => probe.index <= read.resultIndex || !!probe.messageId && probe.messageId === read.messageId;
|
||||
return resources.flatMap(([name, read]) => read && premature(read) ? [`probe preceded resource read: ${name}`] : []);
|
||||
}
|
||||
|
||||
export interface CallerProbe {
|
||||
id: string;
|
||||
charter: string;
|
||||
@@ -358,10 +363,7 @@ export function validateCallerEvidence(input: {
|
||||
else {
|
||||
checkpointProbes.push({ command: String(tool.input.command), observed: probe, index: tool.index });
|
||||
if (parentRead && (tool.index <= parentRead.resultIndex || tool.messageId && tool.messageId === parentRead.messageId)) errors.push(`probe preceded parent entrypoint: ${probe.id}`);
|
||||
for (const id of ['exploratory', 'system-functional']) {
|
||||
const resource = readOf(`/qa/sections/${id}.md`);
|
||||
if (resource && (tool.index <= resource.resultIndex || tool.messageId && tool.messageId === resource.messageId)) errors.push(`probe preceded resource read: ${id}`);
|
||||
}
|
||||
errors.push(...callerProbeReadErrors(tool, ['exploratory', 'system-functional'].map(id => [id, readOf(`/qa/sections/${id}.md`)] as const)));
|
||||
if (input.requireGuardedSmoke) {
|
||||
const command = String(tool.input.command);
|
||||
const guarded = callerDeadlineCommand(command, deadline);
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import { validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const MODELS = path.join(ROOT, 'bin', 'gstack-models');
|
||||
const CONFIG = path.join(ROOT, 'bin', 'gstack-config');
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function fixture(config?: string) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack models cli '));
|
||||
roots.push(root);
|
||||
const state = path.join(root, 'state');
|
||||
const home = path.join(root, 'home');
|
||||
fs.mkdirSync(state);
|
||||
fs.mkdirSync(home);
|
||||
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
return { root, state, home, env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } as Record<string, string> };
|
||||
}
|
||||
|
||||
function models(args: string[], env: Record<string, string>, cwd = path.dirname(env.HOME)) {
|
||||
const result = spawnSync(process.execPath, [MODELS, ...args], { encoding: 'utf-8', env, cwd, timeout: 30_000 });
|
||||
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
|
||||
}
|
||||
|
||||
function config(args: string[], env: Record<string, string>) {
|
||||
const result = spawnSync('bash', [CONFIG, ...args], { encoding: 'utf-8', env, timeout: 30_000 });
|
||||
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
|
||||
}
|
||||
|
||||
function json(args: string[], env: Record<string, string>, cwd?: string) {
|
||||
const result = models([...args, '--json'], env, cwd);
|
||||
expect(result.stderr).toBe('');
|
||||
return { code: result.code, doc: JSON.parse(result.stdout) };
|
||||
}
|
||||
|
||||
function tree(dir: string): string[] {
|
||||
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
|
||||
}
|
||||
|
||||
describe('gstack-models inspector', () => {
|
||||
test('--help explains roles, values, offline behavior, delegation and exits', () => {
|
||||
const result = models(['--help'], fixture().env);
|
||||
expect(result.code).toBe(0);
|
||||
for (const text of ['plan-review', 'implementation', 'frontier | smart | host', 'frontier | smart ', 'offline', 'no paid model call', 'host-controlled', 'Exit: 0', '2 invalid usage', 'schemaVersion 1']) {
|
||||
expect(result.stdout).toContain(text);
|
||||
}
|
||||
});
|
||||
|
||||
test('list is the default and emits one JSON document with both roles and tier entries', () => {
|
||||
const { env, state } = fixture();
|
||||
const human = models([], env);
|
||||
expect(human.code).toBe(0);
|
||||
expect(human.stdout).toContain('plan-review via openai: gpt-6-astra');
|
||||
expect(human.stdout).toContain('implementation via anthropic: claude-opus-5-5');
|
||||
expect(human.stdout).toContain('gstack-config set plan_review_tier smart');
|
||||
expect(human.stderr).toBe('');
|
||||
expect(models(['list'], env).stdout).toBe(human.stdout);
|
||||
|
||||
const { code, doc } = json([], env);
|
||||
expect(code).toBe(0);
|
||||
expect(doc).toMatchObject({ schemaVersion: 1, command: 'list', ok: true, offline: true, configPath: path.join(state, 'config.yaml'), errors: [] });
|
||||
expect(doc.catalog).toMatchObject({ version: 1, sha256: modelCatalogSha256() });
|
||||
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/anthropic', 'plan-review/openai', 'implementation/anthropic', 'implementation/openai']);
|
||||
expect(doc.tiers).toHaveLength(4);
|
||||
expect(doc.settings.plan_review_tier).toEqual({ value: null, effective: 'frontier', origin: 'default' });
|
||||
expect(doc.settings.model_smart_openai).toEqual({ value: null, effective: 'gpt-6.1-sol', origin: 'default' });
|
||||
});
|
||||
|
||||
test('resolve plan-review returns one tagged selection with its source', () => {
|
||||
const { env } = fixture('model_frontier_openai: gpt-pinned\n');
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CLAUDE_MODEL: 'claude-ignored' });
|
||||
expect(code).toBe(0);
|
||||
expect(doc.selections).toHaveLength(1);
|
||||
expect(doc.selections[0]).toMatchObject({ status: 'selected', role: 'plan-review', provider: 'openai', tier: 'frontier', requestedModel: 'gpt-pinned', source: { kind: 'config', key: 'model_frontier_openai' } });
|
||||
const envWins = json(['resolve', '--role=plan-review', '--provider=openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt-env' });
|
||||
expect(envWins.doc.selections[0]).toMatchObject({ requestedModel: 'gpt-env', source: { kind: 'env', name: 'GSTACK_CODEX_MODEL' } });
|
||||
});
|
||||
|
||||
test('resolve implementation returns both providers unless one is named', () => {
|
||||
const { env } = fixture('implementation_tier: frontier\n');
|
||||
const both = json(['resolve', '--role', 'implementation'], env);
|
||||
expect(both.code).toBe(0);
|
||||
expect(both.doc.selections.map((s: { provider: string; requestedModel: string }) => [s.provider, s.requestedModel])).toEqual([['anthropic', 'claude-fable-5-1'], ['openai', 'gpt-6-astra']]);
|
||||
const one = json(['resolve', '--role', 'implementation', '--provider', 'openai'], env);
|
||||
expect(one.doc.selections).toHaveLength(1);
|
||||
expect(one.doc.selections[0].provider).toBe('openai');
|
||||
});
|
||||
|
||||
test('host mode is a valid delegated resolution with a null model', () => {
|
||||
const { env } = fixture('plan_review_tier: host\n');
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
|
||||
expect(code).toBe(0);
|
||||
expect(doc.selections[0]).toMatchObject({ status: 'delegated-host', requestedModel: null, source: { kind: 'host', reason: 'host-mode' } });
|
||||
const human = models(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
|
||||
expect(human.code).toBe(0);
|
||||
expect(human.stdout).toContain('host-controlled');
|
||||
});
|
||||
|
||||
test.each([
|
||||
[['bogus'], "unknown command 'bogus'"],
|
||||
[['resolve'], 'resolve needs --role'],
|
||||
[['resolve', '--role', 'coding'], "unknown role 'coding'"],
|
||||
[['resolve', '--role', 'plan-review'], 'needs --provider'],
|
||||
[['resolve', '--role', 'plan-review', '--provider', 'claude'], "unknown provider 'claude'"],
|
||||
[['resolve', '--role', 'implementation', '--provider', 'OpenAI'], "unknown provider 'OpenAI'"],
|
||||
[['resolve', '--role', 'plan-review', '--role', 'implementation', '--provider', 'openai'], '--role given twice'],
|
||||
[['resolve', '--role'], '--role needs a value'],
|
||||
[['list', '--provider', 'openai'], 'unknown argument'],
|
||||
])('usage error %p exits 2', (args, message) => {
|
||||
const { env } = fixture();
|
||||
const human = models(args, env);
|
||||
expect(human.code).toBe(2);
|
||||
expect(human.stdout).toBe('');
|
||||
expect(human.stderr).toContain(message);
|
||||
const { code, doc } = json(args, env);
|
||||
expect(code).toBe(2);
|
||||
expect(doc).toMatchObject({ schemaVersion: 1, ok: false, selections: [] });
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'usage' });
|
||||
expect(doc.errors[0].problem).toContain(message);
|
||||
});
|
||||
|
||||
test('invalid config exits 1 with structured, source-aware repair', () => {
|
||||
const { env } = fixture('plan_review_tier: fronteir\n');
|
||||
const { code, doc } = json([], env);
|
||||
expect(code).toBe(1);
|
||||
expect(doc).toMatchObject({ ok: false, selections: [], settings: null });
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_plan_review_tier', key: 'plan_review_tier' });
|
||||
expect(doc.errors[0].repair).toContain('gstack-config unset plan_review_tier');
|
||||
expect(doc.errors[0].docs).toContain('#model-policy-config');
|
||||
const human = models(['resolve', '--role', 'implementation'], env);
|
||||
expect(human.code).toBe(1);
|
||||
expect(human.stdout).toBe('');
|
||||
expect(human.stderr).toContain('fix: gstack-config unset plan_review_tier');
|
||||
});
|
||||
|
||||
test('an unusable environment override points at the env var', () => {
|
||||
const { env } = fixture();
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt 6' });
|
||||
expect(code).toBe(1);
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', role: 'plan-review', provider: 'openai' });
|
||||
expect(doc.errors[0].repair[0]).toBe('unset GSTACK_CODEX_MODEL');
|
||||
});
|
||||
|
||||
test('custom routing fails only the affected selections', () => {
|
||||
const { env } = fixture();
|
||||
const { code, doc } = json([], { ...env, CLAUDE_CODE_USE_BEDROCK: '1' });
|
||||
expect(code).toBe(1);
|
||||
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/openai', 'implementation/openai']);
|
||||
expect(doc.errors.map((e: { reason: string; role: string }) => `${e.reason}/${e.role}`)).toEqual(['custom_provider_requires_model/plan-review', 'custom_provider_requires_model/implementation']);
|
||||
});
|
||||
|
||||
test('project settings in the working directory are custom routing; detection limits are published', () => {
|
||||
const { env, root } = fixture();
|
||||
const project = path.join(root, 'project');
|
||||
fs.mkdirSync(path.join(project, '.claude'), { recursive: true });
|
||||
fs.writeFileSync(path.join(project, '.claude', 'settings.local.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } }));
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env, project);
|
||||
expect(code).toBe(1);
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'custom_provider_requires_model', provider: 'anthropic' });
|
||||
expect(doc.providerDetection.unscanned.anthropic.join(' ')).toContain('server-managed');
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env).code).toBe(0);
|
||||
expect(models(['--help'], env).stdout).toContain('cannot see server-managed or MDM policy');
|
||||
});
|
||||
|
||||
test('inspection never writes to the state root or home', () => {
|
||||
const { root, env } = fixture('plan_review_tier: smart\n');
|
||||
const before = tree(root);
|
||||
models([], env);
|
||||
models(['--json'], env);
|
||||
models(['resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], env);
|
||||
models(['resolve', '--role', 'implementation'], env);
|
||||
expect(tree(root)).toEqual(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-config model policy keys', () => {
|
||||
test('defaults, set, resolve again and unset restore', () => {
|
||||
const { env } = fixture();
|
||||
expect(config(['get', 'plan_review_tier'], env)).toMatchObject({ code: 0, stdout: 'frontier' });
|
||||
expect(config(['get', 'implementation_tier'], env)).toMatchObject({ code: 0, stdout: 'smart' });
|
||||
expect(config(['get', 'model_frontier_claude'], env)).toMatchObject({ code: 0, stdout: '' });
|
||||
expect(config(['defaults'], env).stdout).toContain('plan_review_tier:');
|
||||
|
||||
expect(config(['set', 'plan_review_tier', 'smart'], env).code).toBe(0);
|
||||
expect(config(['set', 'model_smart_openai', ' gpt-custom '], env).code).toBe(0);
|
||||
expect(config(['get', 'model_smart_openai'], env).stdout).toBe('gpt-custom');
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'smart', requestedModel: 'gpt-custom', source: { kind: 'config' } });
|
||||
expect(config(['list'], env).stdout).toContain('model_smart_openai: gpt-custom (set)');
|
||||
|
||||
expect(config(['unset', 'plan_review_tier'], env).code).toBe(0);
|
||||
expect(config(['unset', 'model_smart_openai'], env).code).toBe(0);
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'frontier', requestedModel: 'gpt-6-astra', source: { kind: 'catalog' } });
|
||||
});
|
||||
|
||||
test.each([
|
||||
['plan_review_tier', 'fronteir'],
|
||||
['plan_review_tier', ' '],
|
||||
['implementation_tier', 'host'],
|
||||
['model_frontier_openai', 'gpt 6'],
|
||||
['model_frontier_claude', 'a'.repeat(513)],
|
||||
])('invalid %s %p is rejected and the old value kept', (key, value) => {
|
||||
const { env, state } = fixture();
|
||||
const good = key.startsWith('model_') ? 'model-1' : 'frontier';
|
||||
expect(config(['set', key, good], env).code).toBe(0);
|
||||
const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8');
|
||||
const result = config(['set', key, value], env);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain('Existing value left unchanged');
|
||||
expect(result.stderr).toContain(`gstack-config unset ${key}`);
|
||||
expect(result.stderr).toContain('#model-policy-config');
|
||||
expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before);
|
||||
});
|
||||
|
||||
test('set and resolve share one validation rule', () => {
|
||||
const samples: Array<[PolicyConfigKey, string]> = [
|
||||
['plan_review_tier', 'frontier'], ['plan_review_tier', 'smart'], ['plan_review_tier', 'host'], ['plan_review_tier', 'Smart'],
|
||||
['implementation_tier', 'frontier'], ['implementation_tier', 'smart'], ['implementation_tier', 'host'],
|
||||
['model_frontier_claude', 'claude-opus-4-6[1m]'], ['model_frontier_claude', 'vendor+claude@2026'], ['model_frontier_claude', 'a'.repeat(512)],
|
||||
['model_frontier_claude', 'a'.repeat(513)], ['model_smart_claude', 'claude;x'], ['model_smart_claude', 'claude x'], ['model_smart_claude', '$(id)'],
|
||||
['model_frontier_openai', 'gpt-6.1-sol'], ['model_frontier_openai', 'org/model:v1'], ['model_frontier_openai', 'a'.repeat(100)],
|
||||
['model_frontier_openai', 'a'.repeat(101)], ['model_smart_openai', 'gpt+6'], ['model_smart_openai', 'gpt@6'],
|
||||
];
|
||||
const { env } = fixture();
|
||||
for (const [key, value] of samples) {
|
||||
const accepted = config(['set', key, value], env).code === 0;
|
||||
expect({ key, value: value.slice(0, 20), accepted }).toEqual({ key, value: value.slice(0, 20), accepted: validatePolicyValue(key, value) === null });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import { describe, expect, test, spyOn } from 'bun:test';
|
||||
import { collectFreshnessSources, fetchOfficialSource, FRESHNESS_BOUNDS, FRESHNESS_SOURCES, lifecycleDate, parseOfficialSource, parseOfficialSourceBounded, withinDeadline } from '../lib/model-policy-freshness-sources';
|
||||
import { assessFreshness, renderFreshnessReport, storedEvidenceStatus, validEvidence, validLifecycle, validState } from '../lib/model-policy-freshness';
|
||||
import { freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
|
||||
|
||||
describe('official recommendation and lifecycle structures', () => {
|
||||
test('four official fixtures parse, with Mythos and replacement mentions excluded', () => {
|
||||
const observation = freshnessObservation();
|
||||
expect(assessFreshness(observation).status).toBe('current');
|
||||
const anthropic = parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models'));
|
||||
expect(anthropic.recommendations).toEqual({ frontier: 'claude-fable-5-1', smart: 'claude-opus-5-5' });
|
||||
expect(anthropic.lineup).not.toContain('claude-mythos-5-1');
|
||||
const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle'));
|
||||
expect(openai.lifecycle?.map(row => row.modelId)).toEqual(['gpt-5.3-codex', 'gpt-old', 'gpt-unicode-date', 'gpt-iso-date', 'gpt-date-floor']);
|
||||
expect(openai.lifecycle?.find(row => row.modelId === 'gpt-date-floor')).toMatchObject({ deadline: null, supportFloor: '2024-06-13' });
|
||||
});
|
||||
|
||||
test('known-family and actual unknown-family lineup replacements become candidates', () => {
|
||||
const known = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2').replaceAll('Fable 5.1', 'Fable 5.2');
|
||||
const knownReport = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': known }));
|
||||
expect(knownReport.status).toBe('update-candidate');
|
||||
expect(knownReport.candidates[0].reason).toBe('changed-monitored-family-recommendation');
|
||||
const unknown = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'mythos-5-1').replaceAll('Fable 5.1', 'Mythos 5.1');
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': unknown }));
|
||||
expect(report.status).toBe('update-candidate');
|
||||
expect(report.candidates[0]).toMatchObject({ proposed: 'claude-mythos-5-1', reason: 'unknown-replacement-family-needs-triage' });
|
||||
expect(report.complete).toBe(true);
|
||||
});
|
||||
|
||||
test('OpenAI full-catalog newer IDs alone are not recommendations', () => {
|
||||
const original = freshnessFixture('openai-models');
|
||||
expect(assessFreshness(freshnessObservation()).candidates).toEqual([]);
|
||||
const changed = original.replaceAll('gpt-6.1-sol', 'gpt-6.2-sol').replaceAll('GPT-6.1 Sol', 'GPT-6.2 Sol');
|
||||
expect(assessFreshness(freshnessObservation('100', undefined, { 'openai-models': changed })).candidates[0]).toMatchObject({ proposed: 'gpt-6.2-sol' });
|
||||
});
|
||||
|
||||
test('active support floors are never retirement deadlines, even after the floor', () => {
|
||||
const parsed = parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle'));
|
||||
expect(parsed.lifecycle?.[0]).toMatchObject({ state: 'active', deadline: null, supportFloor: '2027-09-01' });
|
||||
expect(assessFreshness(freshnessObservation('100', '2028-10-07T12:00:00.000Z')).state.lifecycle).toEqual([]);
|
||||
});
|
||||
|
||||
test('actual API retirement and removal are urgent, incidental partner prose is ignored', () => {
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(report.status).toBe('update-candidate');
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(renderFreshnessReport(report)).toContain('URGENT');
|
||||
expect(renderFreshnessReport(report)).toContain('Bedrock/Vertex/custom');
|
||||
const removed = assessFreshness(freshnessObservation('100', '2026-11-02T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(removed.state.lifecycle[0].state).toBe('removed');
|
||||
});
|
||||
|
||||
test('OpenAI shutdown rows apply to the deprecated column, not the replacement', () => {
|
||||
const text = freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`');
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'openai-lifecycle': text }));
|
||||
expect(report.state.lifecycle).toHaveLength(1);
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ modelId: 'gpt-6-astra', deadline: '2027-04-01' });
|
||||
});
|
||||
|
||||
test('missing required structures and semantic drift fail closed', () => {
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('Claude API ID', 'Google Cloud ID'))).toThrow('missing-api-id-row');
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('for demanding reasoning', 'for unexplained goals'))).toThrow('unrecognized-recommendation');
|
||||
expect(() => parseOfficialSource('openai-models', freshnessFixture('openai-models').replace('Balance intelligence and cost.', 'Best new model.'))).toThrow('unrecognized-recommendation');
|
||||
expect(() => parseOfficialSource('openai-lifecycle', '# New deprecations\nNo model tables')).toThrow('missing-or-ambiguous-section');
|
||||
expect(() => parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', 'gpt-5.3-codex'))).toThrow('unrecognized-deprecated-model-cell');
|
||||
expect(() => parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle').replace('Active', 'Probably active'))).toThrow('unrecognized-lifecycle-state');
|
||||
expect(() => lifecycleDate('February 30, 2027')).toThrow('malformed-lifecycle-date');
|
||||
expect(() => lifecycleDate('tomorrow')).toThrow('malformed-lifecycle-date');
|
||||
expect(() => parseOfficialSource('anthropic-models', `${freshnessFixture('anthropic-models')}\n## Compare models\nDuplicate`)).toThrow('ambiguous-section');
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('## Using the Models API', 'Use [Other](https://platform.claude.com/docs/en/models/opus-5-5/overview) for demanding reasoning.\n\n## Using the Models API'))).toThrow('unrecognized-recommendation');
|
||||
});
|
||||
});
|
||||
|
||||
describe('bounded fixed-source fetching', () => {
|
||||
test('exact byte limit passes and a streamed byte beyond it fails without trusting content-length', async () => {
|
||||
const atCap = 'x'.repeat(FRESHNESS_BOUNDS.responseBytes);
|
||||
expect((await fetchOfficialSource('openai-models', async () => new Response(atCap))).bytes).toBe(2_097_152);
|
||||
await expect(fetchOfficialSource('openai-models', async () => new Response(`${atCap}x`))).rejects.toThrow('response-too-large');
|
||||
await expect(fetchOfficialSource('openai-models', async () => new Response('x', { headers: { 'content-length': '2097153' } }))).rejects.toThrow('response-too-large');
|
||||
expect(() => parseOfficialSource('openai-models', `${atCap}x`)).toThrow('response-too-large');
|
||||
});
|
||||
|
||||
test('all redirect destinations are checked; three redirects pass, four fail', async () => {
|
||||
const credentialUrl = new URL(FRESHNESS_SOURCES[2].url);
|
||||
credentialUrl.username = 'user';
|
||||
credentialUrl.password = 'pass';
|
||||
for (const location of ['https://evil.example/models.md', 'http://developers.openai.com/api/docs/models.md', credentialUrl.href, 'https://developers.openai.com/api/docs/models.md?untrusted=1']) {
|
||||
let calls = 0;
|
||||
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location } }); })).rejects.toThrow('redirect-not-allowlisted');
|
||||
expect(calls).toBe(1);
|
||||
}
|
||||
let calls = 0;
|
||||
expect((await fetchOfficialSource('openai-models', async () => ++calls <= 3 ? new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }) : new Response('data'))).text).toBe('data');
|
||||
calls = 0;
|
||||
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }); })).rejects.toThrow('redirect-limit');
|
||||
expect(calls).toBe(4);
|
||||
});
|
||||
|
||||
test('per-response and whole-check clocks are installed at the exact approved bounds', async () => {
|
||||
const installed: number[] = [];
|
||||
const real = globalThis.setTimeout;
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, milliseconds: number) => {
|
||||
installed.push(milliseconds);
|
||||
return real(callback, milliseconds === 30_000 ? 5 : milliseconds);
|
||||
}) as typeof setTimeout);
|
||||
try {
|
||||
const result = await collectFreshnessSources(async () => new Promise<Response>(() => {}));
|
||||
expect(result.every(source => source.error === 'deadline-exceeded')).toBe(true);
|
||||
expect(installed.filter(ms => ms === 30_000)).toHaveLength(4);
|
||||
expect(installed).toContain(180_000);
|
||||
} finally { timers.mockRestore(); }
|
||||
const controller = new AbortController();
|
||||
await expect(withinDeadline(new Promise(() => {}), 5, controller)).rejects.toThrow('deadline-exceeded');
|
||||
expect(controller.signal.aborted).toBe(true);
|
||||
});
|
||||
|
||||
test('only four fixed sources are fetched and failures never become parseable success', async () => {
|
||||
const urls: string[] = [];
|
||||
const result = await collectFreshnessSources(async (url, init) => {
|
||||
urls.push(url);
|
||||
expect(init.redirect).toBe('manual');
|
||||
const id = FRESHNESS_SOURCES.find(source => source.url === url)!.id;
|
||||
return id === 'openai-lifecycle' ? new Response('authentication needed', { status: 403 }) : new Response(freshnessFixture(id));
|
||||
});
|
||||
expect(urls.sort()).toEqual(FRESHNESS_SOURCES.map(source => source.url).sort());
|
||||
expect(result.find(source => source.id === 'openai-lifecycle')?.error).toBe('http-403');
|
||||
const observation = freshnessObservation();
|
||||
observation.sources = result;
|
||||
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
|
||||
});
|
||||
|
||||
test('CPU-bound parsing is terminable and the complete-check deadline stops all blocked parsers', async () => {
|
||||
let terminations = 0;
|
||||
const createWorker = () => ({ postMessage() {}, terminate() { terminations++; }, onmessage: null, onerror: null }) as unknown as Worker;
|
||||
const controller = new AbortController();
|
||||
const blocked = parseOfficialSourceBounded('openai-models', freshnessFixture('openai-models'), controller.signal, createWorker);
|
||||
controller.abort();
|
||||
await expect(blocked).rejects.toThrow('deadline-exceeded');
|
||||
expect(terminations).toBe(1);
|
||||
const real = globalThis.setTimeout;
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => real(callback, ms === 180_000 ? 5 : ms)) as typeof setTimeout);
|
||||
try {
|
||||
const result = await collectFreshnessSources(async url => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id)), createWorker);
|
||||
expect(result.every(source => source.error && !source.receipt)).toBe(true);
|
||||
} finally { timers.mockRestore(); }
|
||||
expect(terminations).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('trustworthy evidence and deterministic statuses', () => {
|
||||
test('source-check evidence is separate from catalog qualification and survives artifact expiry', () => {
|
||||
const report = assessFreshness(freshnessObservation());
|
||||
expect(validState(report.state)).toBe(true);
|
||||
expect(validEvidence(report.state.lastSuccess)).toBe(true);
|
||||
expect(report.state.lastSuccess?.nextCheckDueBy).toBe('2026-10-14T12:00:00.000Z');
|
||||
expect(report.state.lastSuccess?.sources).toHaveLength(4);
|
||||
expect(report.state.lastSuccess?.catalog).toEqual(freshnessObservation().catalog);
|
||||
expect(storedEvidenceStatus(freshnessObservation().catalog, JSON.parse(JSON.stringify(report.state.lastSuccess)), '2026-10-08T12:00:00.000Z')).toBe('current');
|
||||
expect(renderFreshnessReport(report)).toContain('not automatic liveness monitoring');
|
||||
});
|
||||
|
||||
test('missing/corrupt/mismatched/future evidence is unknown; thirty-day staleness has an exact boundary', () => {
|
||||
const observation = freshnessObservation();
|
||||
const evidence = assessFreshness(observation).state.lastSuccess!;
|
||||
expect(storedEvidenceStatus(observation.catalog, null, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus({ ...observation.catalog, sha256: 'b'.repeat(64) }, evidence, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, { ...evidence, parserVersion: 99 }, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-10-06T12:00:00.000Z')).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.000Z')).toBe('current');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.001Z')).toBe('stale');
|
||||
});
|
||||
|
||||
test('partial recovery cannot refresh evidence or erase an unresolved retirement', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const next = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'openai-lifecycle': null }), initial.state);
|
||||
expect(next.status).toBe('unknown/source-unavailable');
|
||||
expect(next.state.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
expect(next.state.lifecycle).toEqual(initial.state.lifecycle);
|
||||
const old = assessFreshness(freshnessObservation('102', '2026-11-08T12:00:00.000Z', { 'openai-lifecycle': null }), next.state);
|
||||
expect(old.evidenceStatus).toBe('stale');
|
||||
expect(old.status).toBe('unknown/source-unavailable');
|
||||
const recovered = assessFreshness(freshnessObservation('103', '2026-11-09T12:00:00.000Z'), old.state);
|
||||
expect(recovered.status).toBe('update-candidate');
|
||||
expect(recovered.state.lifecycle).toEqual(old.state.lifecycle);
|
||||
});
|
||||
|
||||
test('corrupt duplicate lifecycle keys and source receipts cannot certify current', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(validLifecycle([initial.state.lifecycle[0], initial.state.lifecycle[0]])).toBe(false);
|
||||
const observation = freshnessObservation();
|
||||
observation.sources[0].receipt!.finalUrl = 'https://evil.example/docs';
|
||||
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
|
||||
const evidence = structuredClone(initial.state.lastSuccess!);
|
||||
evidence.sources[0] = evidence.sources[1];
|
||||
expect(validEvidence(evidence)).toBe(false);
|
||||
expect(validState({ ...initial.state, lastSuccess: evidence })).toBe(false);
|
||||
});
|
||||
|
||||
test('a partial run cannot downgrade an unresolved retirement to legacy or extend its deadline', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const legacy = freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active', 'claude-fable-5-1 | Legacy');
|
||||
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': legacy, 'openai-lifecycle': null }), initial.state);
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(report.state.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
});
|
||||
|
||||
test('only a complete check of a reviewed replacement catalog resolves removed identities', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
|
||||
replacement.catalog.models[0].modelId = 'claude-fable-5-2';
|
||||
replacement.catalog.sha256 = 'b'.repeat(64);
|
||||
replacement.sources = freshnessObservation('101', replacement.checkedAt, {
|
||||
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
}).sources;
|
||||
const incomplete = structuredClone(replacement);
|
||||
incomplete.sources[3] = { id: 'openai-lifecycle', error: 'http-503' };
|
||||
expect(assessFreshness(incomplete, initial.state).state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
const complete = assessFreshness(replacement, initial.state);
|
||||
expect(complete.status).toBe('current');
|
||||
expect(complete.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
});
|
||||
|
||||
test('complete catalog rollback reopens the historical retirement even when the current source row is active', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
|
||||
expect(replacement.status).toBe('current');
|
||||
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), replacement.state);
|
||||
expect(rollback.status).toBe('update-candidate');
|
||||
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01', firstObservedAt: initial.state.lifecycle[0].firstObservedAt });
|
||||
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
});
|
||||
|
||||
test('source-unavailable catalog rollback reopens history without refreshing prior replacement evidence', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
|
||||
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), replacement.state);
|
||||
expect(rollback.status).toBe('unknown/source-unavailable');
|
||||
expect(rollback.state.lastSuccess).toEqual(replacement.state.lastSuccess);
|
||||
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,405 @@
|
||||
import { describe, expect, test, spyOn } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { MODEL_CATALOG, modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import { assessFreshness, freshnessCatalogIdentity } from '../lib/model-policy-freshness';
|
||||
import { FRESHNESS_SOURCES, FRESHNESS_BOUNDS, sha256 } from '../lib/model-policy-freshness-sources';
|
||||
import { FRESHNESS_ISSUE_AUTHOR, FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, publishFreshness, readFreshnessIssue, writeFreshnessIssue, type FreshnessGitHub, type TrackingIssue } from '../lib/model-policy-freshness-publication';
|
||||
import { freshnessGitHub } from '../lib/model-policy-freshness-github';
|
||||
import { runModelPolicyFreshness } from '../scripts/model-policy-freshness';
|
||||
import { freshnessCatalog, freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
|
||||
|
||||
function mockGitHub(initial: (Omit<TrackingIssue, 'author'> & { author?: string })[] = []) {
|
||||
const issues: TrackingIssue[] = structuredClone(initial).map(issue => ({ author: FRESHNESS_ISSUE_AUTHOR, ...issue }));
|
||||
const calls: string[] = [];
|
||||
let identity = { defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 };
|
||||
const github: FreshnessGitHub = {
|
||||
async defaultCatalog() { calls.push('catalog'); return identity; },
|
||||
async listIssues() { calls.push('list'); return structuredClone(issues); },
|
||||
async readIssue(number) { calls.push('read'); return structuredClone(issues.find(issue => issue.number === number)!); },
|
||||
async createIssue(body) { calls.push('create'); const issue: TrackingIssue = { number: issues.length + 1, body, state: 'open', author: FRESHNESS_ISSUE_AUTHOR }; issues.push(issue); return structuredClone(issue); },
|
||||
async updateIssue(number, update) { calls.push('update'); Object.assign(issues.find(issue => issue.number === number)!, update); },
|
||||
};
|
||||
return { github, issues, calls, setIdentity(value: typeof identity) { identity = value; } };
|
||||
}
|
||||
|
||||
describe('stable-marker issue ownership and evidence lifecycle', () => {
|
||||
test('healthy bootstrap creates one closed issue, retains evidence when closed, and repeat publication is idempotent', async () => {
|
||||
const api = mockGitHub();
|
||||
const observation = freshnessObservation();
|
||||
const created = await publishFreshness(observation, api.github, 'refs/heads/main');
|
||||
expect(created.action).toBe('created');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
expect(api.issues[0].body).toContain(FRESHNESS_ISSUE_MARKER);
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess?.checkedAt).toBe(observation.checkedAt);
|
||||
const body = api.issues[0].body;
|
||||
expect((await publishFreshness(observation, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
const next = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
|
||||
expect((await publishFreshness(next, api.github, 'refs/heads/main')).action).toBe('updated');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
});
|
||||
|
||||
test('marker owns the issue, title collisions and human body/disposition text are untouched', async () => {
|
||||
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
const humanPrefix = 'Human notes: retain this default until qualification.\n';
|
||||
const humanSuffix = '\nHuman disposition: mitigation pending; do not delete.\n';
|
||||
const api = mockGitHub([
|
||||
{ number: 40, body: 'Model-policy freshness: advisory maintenance', state: 'open' },
|
||||
{ number: 41, body: humanPrefix + initial + humanSuffix, state: 'closed' },
|
||||
]);
|
||||
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].body).toBe('Model-policy freshness: advisory maintenance');
|
||||
expect(api.issues[1].body.startsWith(humanPrefix)).toBe(true);
|
||||
expect(api.issues[1].body.endsWith(humanSuffix)).toBe(true);
|
||||
expect(api.issues[1].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[1].body).lifecycle[0].state).toBe('deprecated');
|
||||
});
|
||||
|
||||
test('retirement survives source failure and partial recovery; issue never closes on either', async () => {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
const evidence = readFreshnessIssue(api.issues[0].body).state!.lastSuccess;
|
||||
const failure = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(failure.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess).toEqual(evidence);
|
||||
await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].deadline).toBe('2026-11-01');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
});
|
||||
|
||||
test('both complete and source-unavailable catalog rollbacks reopen the closed replacement issue', async () => {
|
||||
for (const unavailable of [false, true]) {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
const replacement = freshnessReplacementObservation();
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 });
|
||||
await publishFreshness(replacement, api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBe(replacement.catalog.sha256);
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
const rollback = freshnessObservation('102', '2026-10-09T12:00:00.000Z', unavailable ? { 'anthropic-lifecycle': null, 'openai-lifecycle': null } : {});
|
||||
const result = await publishFreshness(rollback, api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'update-candidate');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
const stored = readFreshnessIssue(api.issues[0].body);
|
||||
expect(stored.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(stored.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
if (unavailable) expect(stored.state?.lastSuccess?.catalog.sha256).toBe(replacement.catalog.sha256);
|
||||
}
|
||||
});
|
||||
|
||||
test('corrupt state salvages independent lifecycle history and requires explicit manual repair', async () => {
|
||||
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })));
|
||||
const broken = initial.replace(/(<!-- gstack:model-policy-freshness:state:begin -->\n)[^\n]+/, '$1{broken');
|
||||
const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]);
|
||||
expect(readFreshnessIssue(api.issues[0].body).state).toBeNull();
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle).toHaveLength(1);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(result.report?.state.lastSuccess).toBeNull();
|
||||
expect(result.report?.state.lifecycle).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues[0].body).toContain('Human retain decision');
|
||||
expect(api.issues[0].body).toContain('Human footer');
|
||||
expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64'));
|
||||
const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(next.report?.state.recoveryRequired).toBe(true);
|
||||
});
|
||||
|
||||
test('missing machine state is recovery, malformed owned boundary is not overwritten, and output is bounded', async () => {
|
||||
const api = mockGitHub([{ number: 9, body: `${FRESHNESS_ISSUE_MARKER}\nMaintainer notes`, state: 'closed' }]);
|
||||
const result = await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues[0].body).toContain('Maintainer notes');
|
||||
expect(() => writeFreshnessIssue(`${FRESHNESS_ISSUE_MARKER}\n${FRESHNESS_REGION_BEGIN}\nbroken`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
|
||||
expect(() => writeFreshnessIssue('x'.repeat(FRESHNESS_BOUNDS.issueBytes), assessFreshness(freshnessObservation()))).toThrow('tracking-issue-size-limit');
|
||||
expect(() => writeFreshnessIssue(`${FRESHNESS_REGION_END}\n${FRESHNESS_REGION_BEGIN}`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
|
||||
});
|
||||
|
||||
test('duplicate markers fail visibly rather than selecting an arbitrary issue', async () => {
|
||||
const api = mockGitHub([{ number: 1, body: FRESHNESS_ISSUE_MARKER, state: 'open' }, { number: 2, body: FRESHNESS_ISSUE_MARKER, state: 'closed' }]);
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('duplicate-tracking-marker');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(api.calls).not.toContain('create');
|
||||
});
|
||||
|
||||
test('user-created marker copies never become authority or trigger replacement bootstrap', async () => {
|
||||
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
for (const includeOwned of [false, true]) {
|
||||
const api = mockGitHub([{ number: 1, body, state: 'closed', author: 'untrusted-user' }, ...(includeOwned ? [{ number: 2, body, state: 'closed' as const }] : [])]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('untrusted-tracking-marker-owner');
|
||||
expect(api.calls).not.toContain('read');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(api.calls).not.toContain('create');
|
||||
}
|
||||
});
|
||||
|
||||
test('CRLF and CR-only marker lines retain retirement history and publish to the same issue on source failure', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
for (const newline of ['\r\n', '\r']) {
|
||||
const prefix = `Human prefix remains byte-identical.${newline}${newline}`;
|
||||
const suffix = `${newline}${newline}Human suffix remains byte-identical.${newline}`;
|
||||
const body = prefix + writeFreshnessIssue('', initial).replaceAll('\n', newline) + suffix;
|
||||
const api = mockGitHub([{ number: 17, body, state: 'open' }]);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(result.action).toBe('updated');
|
||||
expect(result.issueNumber).toBe(17);
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
|
||||
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
|
||||
const stored = readFreshnessIssue(api.issues[0].body);
|
||||
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(stored.state?.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
}
|
||||
});
|
||||
|
||||
test('inline and quoted marker mentions preserve human prefix and suffix outside the real region', async () => {
|
||||
const initial = assessFreshness(freshnessObservation());
|
||||
const prefix = `Human note mentioning ${FRESHNESS_REGION_BEGIN} inline; preserve this entire sentence.\r\n> ${FRESHNESS_REGION_END}\r\nQuoted '${FRESHNESS_REGION_BEGIN}' and '${FRESHNESS_REGION_END}' are human prose.\r\n\r\n`;
|
||||
const suffix = `\r\nHuman suffix mentions ${FRESHNESS_REGION_END} inline.\r\n> ${FRESHNESS_REGION_BEGIN}\r\n`;
|
||||
const body = prefix + writeFreshnessIssue('', initial) + suffix;
|
||||
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), initial.state);
|
||||
const rewritten = writeFreshnessIssue(body, report);
|
||||
expect(rewritten.startsWith(prefix)).toBe(true);
|
||||
expect(rewritten.endsWith(suffix)).toBe(true);
|
||||
expect(readFreshnessIssue(rewritten).state).toEqual(report.state);
|
||||
const api = mockGitHub([{ number: 19, body, state: 'closed' }]);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(result.action).toBe('updated');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
|
||||
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
|
||||
});
|
||||
|
||||
test('malformed or missing top markers with owned evidence never become a healthy bootstrap', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
for (const marker of ['', `${FRESHNESS_ISSUE_MARKER} `, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace('<!-- ', '<!--'), FRESHNESS_ISSUE_MARKER.slice(0, -1)]) {
|
||||
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_ISSUE_MARKER, marker);
|
||||
const api = mockGitHub([{ number: 23, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(readFreshnessIssue(body).lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(readFreshnessIssue(body).recoveryRequired).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('standalone malformed top markers without readable regions block healthy bootstrap', async () => {
|
||||
for (const marker of [` ${FRESHNESS_ISSUE_MARKER}`, `${FRESHNESS_ISSUE_MARKER}\t`, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace(':v1', '-v1')]) {
|
||||
const body = `Human text\r\n${marker}\r\nRetirement history needs manual recovery.`;
|
||||
const api = mockGitHub([{ number: 29, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
}
|
||||
});
|
||||
|
||||
test('malformed owned boundaries salvage available lifecycle history but never certify a healthy publication', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_BEGIN.replace(':begin', ':begn')).replace(FRESHNESS_REGION_END, FRESHNESS_REGION_END.replace(':end', ':en'));
|
||||
const stored = readFreshnessIssue(body);
|
||||
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(stored.recoveryRequired).toBe(true);
|
||||
const api = mockGitHub([{ number: 31, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
});
|
||||
});
|
||||
|
||||
describe('publication ordering and default-branch guards', () => {
|
||||
test('both completion orders retain the newer unresolved retirement; old rerun is rejected even with a new observation date', async () => {
|
||||
for (const newerFirst of [false, true]) {
|
||||
const api = mockGitHub();
|
||||
const older = freshnessObservation('100');
|
||||
const newer = freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() });
|
||||
for (const observation of newerFirst ? [newer, older] : [older, newer]) await publishFreshness(observation, api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('101');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].modelId).toBe('claude-fable-5-1');
|
||||
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
|
||||
rerun.run.attempt = 2;
|
||||
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
}
|
||||
});
|
||||
|
||||
test('higher run ID cannot publish an older check timestamp, but a newer attempt can advance the same run', async () => {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect((await publishFreshness(freshnessObservation('101', '2026-10-07T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
|
||||
rerun.run.attempt = 2;
|
||||
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('updated');
|
||||
});
|
||||
|
||||
test('non-main is report-only with zero GitHub operations; a non-main default disables writes', async () => {
|
||||
const api = mockGitHub();
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/feature')).action).toBe('report-only');
|
||||
expect(api.calls).toEqual([]);
|
||||
api.setIdentity({ defaultBranch: 'develop', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('default-branch-is-not-main');
|
||||
expect(api.calls).not.toContain('create');
|
||||
});
|
||||
|
||||
test('obsolete local catalog and catalog change immediately before write are rejected', async () => {
|
||||
const api = mockGitHub();
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: 'b'.repeat(64) });
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
|
||||
expect(api.calls).not.toContain('list');
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
api.github.listIssues = async () => { api.setIdentity({ defaultBranch: 'main', sourceSha256: 'c'.repeat(64) }); return []; };
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('final issue read preserves human text added during the authoritative catalog check', async () => {
|
||||
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
const api = mockGitHub([{ number: 1, body, state: 'closed' }]);
|
||||
const authority = api.github.defaultCatalog;
|
||||
api.github.defaultCatalog = async signal => {
|
||||
const identity = await authority(signal);
|
||||
if (api.calls.includes('list')) api.issues[0].body += '\nLate maintainer note';
|
||||
return identity;
|
||||
};
|
||||
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].body.endsWith('\nLate maintainer note')).toBe(true);
|
||||
});
|
||||
|
||||
test('final issue read rejects an older observation when newer authoritative history arrives during the catalog check', async () => {
|
||||
const api = mockGitHub([{ number: 1, body: writeFreshnessIssue('', assessFreshness(freshnessObservation())), state: 'closed' }]);
|
||||
const authority = api.github.defaultCatalog;
|
||||
api.github.defaultCatalog = async signal => {
|
||||
const identity = await authority(signal);
|
||||
if (api.calls.includes('list')) api.issues[0] = { ...api.issues[0], state: 'open', body: writeFreshnessIssue(api.issues[0].body, assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }))) };
|
||||
return identity;
|
||||
};
|
||||
expect((await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('102');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].state).toBe('deprecated');
|
||||
});
|
||||
|
||||
test('publication has a separate exact sixty-second deadline and aborts the blocked API', async () => {
|
||||
const real = globalThis.setTimeout;
|
||||
const installed: number[] = [];
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => { installed.push(ms); return real(callback, 5); }) as typeof setTimeout);
|
||||
const api = mockGitHub();
|
||||
let signal: AbortSignal | undefined;
|
||||
api.github.defaultCatalog = async input => { signal = input; return new Promise(() => {}); };
|
||||
try { await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('deadline-exceeded'); }
|
||||
finally { timers.mockRestore(); }
|
||||
expect(installed).toEqual([60_000]);
|
||||
expect(signal?.aborted).toBe(true);
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('network adapter, real catalog and CLI seams', () => {
|
||||
test('real catalog export and canonical hash bind the four identities and separate source-file hash', () => {
|
||||
const source = readFileSync(new URL('../lib/model-catalog.ts', import.meta.url), 'utf8');
|
||||
const identity = freshnessCatalogIdentity(source);
|
||||
expect(identity.sha256).toBe(modelCatalogSha256());
|
||||
expect(identity.sourceSha256).toBe(sha256(source));
|
||||
expect(identity.models).toEqual(MODEL_CATALOG.map(entry => ({ provider: entry.provider, tier: entry.tier, modelId: entry.model })));
|
||||
});
|
||||
|
||||
test('GitHub reads all-state paginated issues, ignores PRs, and compares remote bytes without executing them', async () => {
|
||||
const calls: { url: string; method?: string }[] = [];
|
||||
const text = 'untrusted TS is data, never import or eval';
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async (url, init) => {
|
||||
calls.push({ url, method: init.method });
|
||||
expect(init.redirect).toBe('error');
|
||||
if (url.endsWith('/repos/owner/repo')) return Response.json({ default_branch: 'main' });
|
||||
if (url.includes('/contents/')) return Response.json({ encoding: 'base64', content: Buffer.from(text).toString('base64'), size: Buffer.byteLength(text) });
|
||||
if (new URL(url).searchParams.get('page') === '1') return Response.json(Array.from({ length: 100 }, (_, i) => ({ number: i + 1, body: '', state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR }, ...(i === 0 ? {} : { pull_request: {} }) })));
|
||||
if (new URL(url).searchParams.get('page') === '2') return Response.json([{ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }]);
|
||||
return Response.json({ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } });
|
||||
});
|
||||
const signal = new AbortController().signal;
|
||||
expect(await api.defaultCatalog(signal)).toEqual({ defaultBranch: 'main', sourceSha256: sha256(text) });
|
||||
expect((await api.listIssues(signal)).map(issue => issue.number)).toEqual([1, 101]);
|
||||
await api.updateIssue(101, { body: 'machine-owned data', state: 'open' }, signal);
|
||||
expect(calls.at(-1)?.method).toBe('PATCH');
|
||||
expect(calls.filter(call => call.url.includes('/issues?')).every(call => call.url.includes('state=all'))).toBe(true);
|
||||
expect(() => freshnessGitHub('owner/repo; shell', 'fixture')).toThrow('prerequisites-missing');
|
||||
await expect(freshnessGitHub('owner/repo', 'fixture', async () => new Response(null, { status: 403 })).listIssues(signal)).rejects.toThrow('github-http-403');
|
||||
});
|
||||
|
||||
test('creator-filtered discovery stays one request with more than one hundred unrelated issues and PRs', async () => {
|
||||
const unrelated = Array.from({ length: 301 }, (_, i) => ({ number: i + 1, body: 'Unrelated public issue body', state: 'open', user: { login: 'ordinary-user' }, ...(i % 2 ? { pull_request: {} } : {}) }));
|
||||
const items = [...unrelated, { number: 9000, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }];
|
||||
const requests: URL[] = [];
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async url => {
|
||||
const request = new URL(url);
|
||||
requests.push(request);
|
||||
const creator = request.searchParams.get('creator');
|
||||
const selected = creator ? items.filter(issue => issue.user.login === creator) : items;
|
||||
const page = Number(request.searchParams.get('page'));
|
||||
return Response.json(selected.slice((page - 1) * 100, page * 100));
|
||||
});
|
||||
const found = await api.listIssues(new AbortController().signal);
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests[0].searchParams.get('creator')).toBe(FRESHNESS_ISSUE_AUTHOR);
|
||||
expect(requests[0].searchParams.get('state')).toBe('all');
|
||||
expect(found.map(issue => issue.number)).toEqual([9000]);
|
||||
});
|
||||
|
||||
test('creator filter never replaces returned-owner validation', async () => {
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async () => Response.json([{ number: 7, body: FRESHNESS_ISSUE_MARKER, state: 'open', user: { login: 'untrusted-user' } }]));
|
||||
await expect(api.listIssues(new AbortController().signal)).rejects.toThrow('github-issue-owner-filter-mismatch');
|
||||
});
|
||||
|
||||
test('CLI non-main --publish still has no GitHub calls; main publication failure preserves source receipts and fails honestly', async () => {
|
||||
const api = mockGitHub();
|
||||
const fetcher = async (url: string) => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id));
|
||||
const env = { GITHUB_REF: 'refs/heads/feature', GITHUB_RUN_ID: '100', GITHUB_SHA: 'a'.repeat(40) };
|
||||
const result = await runModelPolicyFreshness({ env, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
|
||||
expect(result.publication?.action).toBe('report-only');
|
||||
expect(api.calls).toEqual([]);
|
||||
api.github.defaultCatalog = async () => { throw new Error('github-http-403'); };
|
||||
const failed = await runModelPolicyFreshness({ env: { ...env, GITHUB_REF: 'refs/heads/main', GITHUB_REPOSITORY: 'owner/repo', GH_TOKEN: 'fixture' }, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
|
||||
expect(failed.publication?.action).toBe('failed');
|
||||
expect(failed.report.status).toBe('unknown/source-unavailable');
|
||||
expect(failed.observation.sources.filter(source => source.receipt)).toHaveLength(4);
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('weekly/manual workflow contract', () => {
|
||||
test('one repository-wide serialized group, advisory schedule, main writes and non-main reports', () => {
|
||||
const text = readFileSync(new URL('../.github/workflows/model-policy-freshness.yml', import.meta.url), 'utf8');
|
||||
const workflow = Bun.YAML.parse(text) as { concurrency: { group: string; 'cancel-in-progress': boolean }; on: { schedule: { cron: string }[]; workflow_dispatch: unknown }; jobs: Record<string, { steps: { name?: string; if?: string; run?: string }[] }> };
|
||||
expect(workflow.concurrency).toEqual({ group: 'model-policy-freshness', 'cancel-in-progress': false });
|
||||
expect(workflow.on.schedule).toEqual([{ cron: '0 11 * * 1' }]);
|
||||
expect(Object.keys(workflow.jobs)).toEqual(['advisory']);
|
||||
expect(text).toContain('name: Model-policy freshness');
|
||||
expect(text).toContain('workflow_dispatch:');
|
||||
expect(text).toContain("cron: '0 11 * * 1'");
|
||||
expect(text.match(/group:/g)).toHaveLength(1);
|
||||
expect(text).toContain('group: model-policy-freshness\n cancel-in-progress: false');
|
||||
expect(text).not.toContain('model-policy-freshness-${{ github.ref }}');
|
||||
expect(text).toContain("if: github.ref == 'refs/heads/main'");
|
||||
expect(text).toContain("if: github.ref != 'refs/heads/main'");
|
||||
expect(text).toContain('run: bun scripts/model-policy-freshness.ts --publish');
|
||||
expect(text).toContain('run: bun scripts/model-policy-freshness.ts\n');
|
||||
expect(text).toContain('if-no-files-found: error');
|
||||
expect(text).toContain('if: always()');
|
||||
expect(text).toContain('bun-version: 1.4.2');
|
||||
expect(text).toContain('persist-credentials: false');
|
||||
expect(text).toContain('actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a');
|
||||
expect(text.match(/secrets\.[A-Z_]+/g)).toEqual(['secrets.GITHUB_TOKEN']);
|
||||
expect(text).not.toMatch(/pull_request|ANTHROPIC_API_KEY|OPENAI_API_KEY|codex exec|claude -p|gh issue|eval:bg|gstack-config set/);
|
||||
expect(text.match(/uses: [^\n]+/g)?.every(line => /@[a-f0-9]{40}$/.test(line))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,524 @@
|
||||
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { catalogEntry, MODEL_CATALOG, modelCatalogCanonicalJson, modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import {
|
||||
ANTHROPIC_MODEL_ID_PATTERN,
|
||||
CUSTOM_PROVIDER_UNSCANNED,
|
||||
describeSelection,
|
||||
detectCustomProvider as detectWith,
|
||||
isValidModelId,
|
||||
ModelPolicyError,
|
||||
OPENAI_MODEL_ID_PATTERN,
|
||||
parseModelPolicyConfig,
|
||||
readModelPolicyConfig,
|
||||
resolveImplementationModels as implementationWith,
|
||||
resolvePlanReviewModel as planReviewWith,
|
||||
selectionRepair,
|
||||
validatePolicyValue,
|
||||
type ModelSelection,
|
||||
} from '../lib/model-policy';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
|
||||
import { CODEX_RUNTIME_MODEL_PATTERN } from '../scripts/resolve-codex-generation-model';
|
||||
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const roots: string[] = [];
|
||||
const ISOLATED_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-native-'));
|
||||
const ISOLATED = { cwd: ISOLATED_DIR, claudeManagedDir: path.join(ISOLATED_DIR, 'managed'), codexSystemConfig: null };
|
||||
const resolvePlanReviewModel = (opts: Parameters<typeof planReviewWith>[0]) => planReviewWith({ ...ISOLATED, ...opts });
|
||||
const resolveImplementationModels = (opts: Parameters<typeof implementationWith>[0]) => implementationWith({ ...ISOLATED, ...opts });
|
||||
const detectCustomProvider = (provider: Parameters<typeof detectWith>[0], opts: Parameters<typeof detectWith>[1] = {}) => detectWith(provider, { ...ISOLATED, ...opts });
|
||||
|
||||
afterAll(() => fs.rmSync(ISOLATED_DIR, { recursive: true, force: true }));
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) {
|
||||
fs.chmodSync(root, 0o700);
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function fixture(config?: string) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-'));
|
||||
roots.push(root);
|
||||
const state = path.join(root, 'state');
|
||||
const home = path.join(root, 'home');
|
||||
fs.mkdirSync(state);
|
||||
fs.mkdirSync(home);
|
||||
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
const env: Record<string, string | undefined> = { HOME: home, GSTACK_STATE_ROOT: state };
|
||||
return { root, state, home, env };
|
||||
}
|
||||
|
||||
function policyError(run: () => unknown): ModelPolicyError {
|
||||
try {
|
||||
run();
|
||||
} catch (error) {
|
||||
if (error instanceof ModelPolicyError) return error;
|
||||
throw error;
|
||||
}
|
||||
throw new Error('expected a ModelPolicyError');
|
||||
}
|
||||
|
||||
function selected(selection: ModelSelection): string {
|
||||
if (selection.status !== 'selected') throw new Error(`expected selected, got ${selection.status}`);
|
||||
return selection.requestedModel;
|
||||
}
|
||||
|
||||
function tree(dir: string): string[] {
|
||||
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
|
||||
}
|
||||
|
||||
describe('model catalog', () => {
|
||||
test('ships exactly the four approved tier IDs with provenance', () => {
|
||||
expect(MODEL_CATALOG.map(e => [e.tier, e.provider, e.model])).toEqual([
|
||||
['frontier', 'anthropic', 'claude-fable-5-1'],
|
||||
['frontier', 'openai', 'gpt-6-astra'],
|
||||
['smart', 'anthropic', 'claude-opus-5-5'],
|
||||
['smart', 'openai', 'gpt-6.1-sol'],
|
||||
]);
|
||||
for (const entry of MODEL_CATALOG) {
|
||||
expect(entry.sourceUrl).toMatch(/^https:\/\/(platform\.claude\.com|developers\.openai\.com)\//);
|
||||
expect(entry.verifiedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
||||
expect(Object.isFrozen(entry)).toBe(true);
|
||||
}
|
||||
expect(Object.isFrozen(MODEL_CATALOG)).toBe(true);
|
||||
expect(catalogEntry('openai', 'smart').model).toBe('gpt-6.1-sol');
|
||||
});
|
||||
|
||||
test('identity is deterministic and independent of user config', () => {
|
||||
const json = modelCatalogCanonicalJson();
|
||||
expect(JSON.parse(json)).toEqual({ version: 1, entries: MODEL_CATALOG.map(e => ({ ...e })) });
|
||||
expect(modelCatalogSha256()).toBe(createHash('sha256').update(json).digest('hex'));
|
||||
expect(modelCatalogCanonicalJson()).toBe(json);
|
||||
const source = fs.readFileSync(path.join(ROOT, 'lib/model-catalog.ts'), 'utf-8');
|
||||
expect(source).not.toMatch(/process\.env|state-root|readFileSync|config\.yaml/);
|
||||
});
|
||||
|
||||
test('eval rulers and the no-role Codex pattern stay independent of the catalog', () => {
|
||||
expect(CLAUDE_FRONTIER_EVAL_MODEL).toBe('claude-fable-5-1');
|
||||
expect(CODEX_FRONTIER_MODEL).toBe('gpt-6-astra');
|
||||
for (const file of ['lib/eval-model.ts', 'scripts/resolvers/constants.ts']) {
|
||||
expect(fs.readFileSync(path.join(ROOT, file), 'utf-8')).not.toMatch(/from\s+['"][^'"]*model-(catalog|policy)/);
|
||||
}
|
||||
expect(OPENAI_MODEL_ID_PATTERN.source).toBe(CODEX_RUNTIME_MODEL_PATTERN.source);
|
||||
});
|
||||
});
|
||||
|
||||
describe('model ID validation', () => {
|
||||
test.each([
|
||||
['claude-opus-5-5', true],
|
||||
['us.anthropic.claude-opus-5-5-v1:0', true],
|
||||
['arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude', true],
|
||||
['claude-opus-4@20250514', true],
|
||||
['claude-opus-4-6[1m]', true],
|
||||
['vendor+claude', true],
|
||||
['a'.repeat(512), true],
|
||||
['a'.repeat(513), false],
|
||||
['claude opus', false],
|
||||
['claude\topus', false],
|
||||
['claude\u0007', false],
|
||||
['$(touch x)', false],
|
||||
['claude;rm', false],
|
||||
['', false],
|
||||
])('anthropic %p => %p', (value, ok) => {
|
||||
expect(isValidModelId('anthropic', value)).toBe(ok);
|
||||
expect(ANTHROPIC_MODEL_ID_PATTERN.test(value)).toBe(ok);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['gpt-6.1-sol', true],
|
||||
['org/model:v1', true],
|
||||
['a'.repeat(100), true],
|
||||
['a'.repeat(101), false],
|
||||
['gpt+6', false],
|
||||
['gpt 6', false],
|
||||
['gpt"6', false],
|
||||
])('openai %p => %p', (value, ok) => {
|
||||
expect(isValidModelId('openai', value)).toBe(ok);
|
||||
});
|
||||
|
||||
test('policy values are trimmed and empty values fail', () => {
|
||||
expect(validatePolicyValue('model_smart_claude', ' claude-x ')).toBeNull();
|
||||
expect(validatePolicyValue('plan_review_tier', ' host ')).toBeNull();
|
||||
expect(validatePolicyValue('model_smart_claude', ' ')?.reason).toBe('empty_value');
|
||||
expect(validatePolicyValue('implementation_tier', 'host')?.reason).toBe('invalid_implementation_tier');
|
||||
expect(validatePolicyValue('plan_review_tier', 'fronteir')?.reason).toBe('invalid_plan_review_tier');
|
||||
expect(validatePolicyValue('model_frontier_openai', 'gpt+6')?.reason).toBe('invalid_model_id');
|
||||
});
|
||||
});
|
||||
|
||||
describe('strict policy config snapshot', () => {
|
||||
test('a missing file or absent keys give the documented defaults', () => {
|
||||
const { env } = fixture();
|
||||
const config = readModelPolicyConfig({ env });
|
||||
expect(config.exists).toBe(false);
|
||||
expect(config.planReviewTier).toMatchObject({ value: 'frontier', origin: 'default' });
|
||||
expect(config.implementationTier).toMatchObject({ value: 'smart', origin: 'default' });
|
||||
expect(config.overrides).toEqual({ anthropic: { frontier: null, smart: null }, openai: { frontier: null, smart: null } });
|
||||
expect(readModelPolicyConfig({ env: fixture('telemetry: off\n').env }).exists).toBe(true);
|
||||
});
|
||||
|
||||
test('last value wins, CRLF and comments are tolerated, values are trimmed', () => {
|
||||
const config = parseModelPolicyConfig('# plan_review_tier: host\r\nplan_review_tier: host\r\nplan_review_tier: smart \r\nmodel_smart_openai: gpt-x\r\n', '/c.yaml');
|
||||
expect(config.planReviewTier).toMatchObject({ value: 'smart', origin: 'config', path: '/c.yaml' });
|
||||
expect(config.overrides.openai.smart).toBe('gpt-x');
|
||||
});
|
||||
|
||||
test.each([
|
||||
['plan_review_tier:\n', 'empty_value', 'plan_review_tier'],
|
||||
['plan_review_tier: fronteir\n', 'invalid_plan_review_tier', 'plan_review_tier'],
|
||||
['implementation_tier: host\n', 'invalid_implementation_tier', 'implementation_tier'],
|
||||
['model_frontier_openai: gpt 6\n', 'invalid_model_id', 'model_frontier_openai'],
|
||||
[' plan_review_tier: smart\n', 'config_malformed', 'plan_review_tier'],
|
||||
['model_smart_claude : claude-x\n', 'config_malformed', 'model_smart_claude'],
|
||||
])('%p fails with %p', (text, reason, key) => {
|
||||
const error = policyError(() => parseModelPolicyConfig(text, '/c.yaml'));
|
||||
expect(error.reason).toBe(reason as ModelPolicyError['reason']);
|
||||
expect(error.key).toBe(key);
|
||||
expect(error.repair.join('\n')).toContain(`gstack-config unset ${key}`);
|
||||
expect(error.docs).toContain('troubleshooting.md#model-policy-config');
|
||||
for (const step of error.repair) expect(step).not.toMatch(/[|]/);
|
||||
});
|
||||
|
||||
test('an invalid unused override still fails closed', () => {
|
||||
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_claude: bad value\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('invalid_model_id');
|
||||
});
|
||||
|
||||
test('a directory, an unreadable file or a file state root is an error, never the default', () => {
|
||||
const dir = fixture();
|
||||
fs.mkdirSync(path.join(dir.state, 'config.yaml'));
|
||||
const isDir = policyError(() => readModelPolicyConfig({ env: dir.env }));
|
||||
expect(isDir).toMatchObject({ reason: 'config_unreadable', cause: 'config.yaml is a directory' });
|
||||
|
||||
const notDir = fixture();
|
||||
const fileRoot = path.join(notDir.root, 'file-root');
|
||||
fs.writeFileSync(fileRoot, '');
|
||||
expect(policyError(() => readModelPolicyConfig({ env: { ...notDir.env, GSTACK_STATE_ROOT: fileRoot } })).reason).toBe('config_unreadable');
|
||||
|
||||
if (process.platform !== 'win32' && process.getuid?.() !== 0) {
|
||||
const denied = fixture('plan_review_tier: smart\n');
|
||||
fs.chmodSync(path.join(denied.state, 'config.yaml'), 0o000);
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env: denied.env }))).toMatchObject({ reason: 'config_unreadable', cause: 'permission denied' });
|
||||
}
|
||||
});
|
||||
|
||||
test('a bound snapshot is used for the whole resolution even if the file is replaced', () => {
|
||||
const { state, env } = fixture('model_frontier_openai: gpt-old\n');
|
||||
const config = readModelPolicyConfig({ env });
|
||||
const next = path.join(state, 'config.yaml.next');
|
||||
fs.writeFileSync(next, 'plan_review_tier: smart\nmodel_frontier_openai: gpt-new\n');
|
||||
fs.renameSync(next, path.join(state, 'config.yaml'));
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', env, config }))).toBe('gpt-old');
|
||||
const fresh = resolvePlanReviewModel({ provider: 'openai', env });
|
||||
expect(fresh.tier).toBe('smart');
|
||||
expect(selected(fresh)).toBe('gpt-6.1-sol');
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan-review precedence', () => {
|
||||
test.each(['anthropic', 'openai'] as const)('%s: request > env > per-tier config > catalog', provider => {
|
||||
const envName = provider === 'anthropic' ? 'GSTACK_CLAUDE_MODEL' : 'GSTACK_CODEX_MODEL';
|
||||
const key = provider === 'anthropic' ? 'model_frontier_claude' : 'model_frontier_openai';
|
||||
const { env, state } = fixture(`${key}: tier-model\n`);
|
||||
const withEnv = { ...env, [envName]: ' env-model ' };
|
||||
|
||||
const request = resolvePlanReviewModel({ provider, requestedModel: 'request-model', env: withEnv });
|
||||
expect(request).toMatchObject({ status: 'selected', role: 'plan-review', provider, tier: 'frontier', requestedModel: 'request-model', source: { kind: 'request' } });
|
||||
|
||||
const fromEnv = resolvePlanReviewModel({ provider, env: withEnv });
|
||||
expect(fromEnv).toMatchObject({ requestedModel: 'env-model', source: { kind: 'env', name: envName } });
|
||||
|
||||
const fromConfig = resolvePlanReviewModel({ provider, env });
|
||||
expect(fromConfig).toMatchObject({ requestedModel: 'tier-model', source: { kind: 'config', key, path: path.join(state, 'config.yaml') } });
|
||||
|
||||
const fromCatalog = resolvePlanReviewModel({ provider, env: fixture().env });
|
||||
expect(fromCatalog).toMatchObject({ requestedModel: catalogEntry(provider, 'frontier').model, source: { kind: 'catalog', catalogVersion: 1, verifiedAt: '2026-10-07' } });
|
||||
expect(fromCatalog.tierFrom).toMatchObject({ key: 'plan_review_tier', origin: 'default' });
|
||||
});
|
||||
|
||||
test('smart tier reads the smart override, not the frontier one', () => {
|
||||
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_openai: gpt-frontier-pin\n');
|
||||
const selection = resolvePlanReviewModel({ provider: 'openai', env });
|
||||
expect(selection).toMatchObject({ tier: 'smart', requestedModel: 'gpt-6.1-sol', tierFrom: { origin: 'config' } });
|
||||
});
|
||||
|
||||
test('host mode delegates with a null model; request and env still win; tier overrides do not apply', () => {
|
||||
const { env } = fixture('plan_review_tier: host\nmodel_frontier_claude: claude-pin\nmodel_smart_claude: claude-pin\n');
|
||||
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env });
|
||||
expect(delegated).toMatchObject({ status: 'delegated-host', requestedModel: null, tier: 'host', catalog: null, source: { kind: 'host', reason: 'host-mode', key: 'plan_review_tier' } });
|
||||
expect(describeSelection(delegated)).toContain('host-controlled');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-env' } }))).toBe('claude-env');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', requestedModel: 'gpt-req', env }))).toBe('gpt-req');
|
||||
});
|
||||
|
||||
test('an invalid environment override names the env var and its repair, not a tier', () => {
|
||||
const { env } = fixture();
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt"; rm -rf /' } }));
|
||||
expect(error).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', source: 'env' });
|
||||
expect(error.repair[0]).toBe('unset GSTACK_CODEX_MODEL');
|
||||
expect(error.docs).toContain('#model-policy-selection');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', requestedModel: 'two words', env })).key).toBe('request');
|
||||
});
|
||||
|
||||
test('an empty env override is unset, matching the legacy readers', () => {
|
||||
const { env } = fixture();
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: '' } }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('selectionRepair follows the winning source', () => {
|
||||
const { env } = fixture();
|
||||
expect(selectionRepair(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toEqual(['unset GSTACK_CODEX_MODEL', 'export GSTACK_CODEX_MODEL=<model-id>']);
|
||||
const catalogRepair = selectionRepair(resolvePlanReviewModel({ provider: 'anthropic', env }));
|
||||
expect(catalogRepair).toContain('gstack-config set model_frontier_claude <model-id>');
|
||||
expect(catalogRepair).toContain('gstack-config set plan_review_tier host');
|
||||
});
|
||||
});
|
||||
|
||||
describe('custom-provider guard', () => {
|
||||
test.each(['CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY'])('%s blocks the public default but not explicit choices', flag => {
|
||||
const { env } = fixture();
|
||||
for (const value of ['1', 'true', 'YES', 'on']) {
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }));
|
||||
expect(error).toMatchObject({ reason: 'custom_provider_requires_model', source: 'native' });
|
||||
expect(error.cause).toContain(flag);
|
||||
expect(error.repair).toEqual(['gstack-config set model_frontier_claude <model-id-your-endpoint-serves>', 'export GSTACK_CLAUDE_MODEL=<model-id-your-endpoint-serves>', 'gstack-config set plan_review_tier host']);
|
||||
expect(error.docs).toContain('#model-policy-provider');
|
||||
}
|
||||
for (const value of ['0', 'false', '', 'no']) {
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }).source.kind).toBe('catalog');
|
||||
}
|
||||
const custom = { ...env, [flag]: '1' };
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...custom, GSTACK_CLAUDE_MODEL: 'us.anthropic.claude-x-v1:0' } }))).toBe('us.anthropic.claude-x-v1:0');
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('plan_review_tier: host\n').env, [flag]: '1' } }).status).toBe('delegated-host');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('model_frontier_claude: vertex-claude@1\n').env, [flag]: '1' } }))).toBe('vertex-claude@1');
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', env: custom }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('ANTHROPIC_BASE_URL counts only when non-default and is never printed', () => {
|
||||
const { env } = fixture();
|
||||
expect(detectCustomProvider('anthropic', { env: { ...env, ANTHROPIC_BASE_URL: 'https://api.anthropic.com/' } }).custom).toBe(false);
|
||||
const endpoint = new URL('https://proxy.example/v1');
|
||||
endpoint.username = 'fixture-user';
|
||||
endpoint.password = 'fixture-password';
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, ANTHROPIC_BASE_URL: endpoint.href } }));
|
||||
expect(error.reason).toBe('custom_provider_requires_model');
|
||||
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.password);
|
||||
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.username);
|
||||
});
|
||||
|
||||
test('Claude settings.json env signals count; malformed settings are unresolved', () => {
|
||||
const { env, home } = fixture();
|
||||
fs.mkdirSync(path.join(home, '.claude'));
|
||||
const settings = path.join(home, '.claude', 'settings.json');
|
||||
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } }));
|
||||
expect(detectCustomProvider('anthropic', { env }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${settings})`]);
|
||||
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '0' }, model: 'opus' }));
|
||||
expect(detectCustomProvider('anthropic', { env }).custom).toBe(false);
|
||||
fs.writeFileSync(settings, '{not json');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env }))).toMatchObject({ reason: 'native_provider_unresolved', key: settings });
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-x' } }))).toBe('claude-x');
|
||||
const other = path.join(home, 'alt');
|
||||
fs.mkdirSync(other);
|
||||
fs.writeFileSync(path.join(other, 'settings.json'), JSON.stringify({ env: { CLAUDE_CODE_USE_VERTEX: 'true' } }));
|
||||
expect(detectCustomProvider('anthropic', { env: { ...env, CLAUDE_CONFIG_DIR: other } }).custom).toBe(true);
|
||||
});
|
||||
|
||||
test('Codex model_provider, profile inheritance and base URLs', () => {
|
||||
const { env, home } = fixture();
|
||||
const codex = path.join(home, '.codex');
|
||||
fs.mkdirSync(codex);
|
||||
const write = (toml: string) => fs.writeFileSync(path.join(codex, 'config.toml'), toml);
|
||||
const custom = () => detectCustomProvider('openai', { env }).custom;
|
||||
|
||||
expect(custom()).toBe(false);
|
||||
write('model = "gpt-5.5"\nmodel_provider = "openai"\n');
|
||||
expect(custom()).toBe(false);
|
||||
write('model_provider = "azure"\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('custom_provider_requires_model');
|
||||
write('model_provider = "azure"\nprofile = "work"\n[profiles.work]\nmodel_provider = "openai"\n');
|
||||
expect(custom()).toBe(false);
|
||||
write('profile = "work"\n[profiles.work]\nmodel_provider = "ollama"\n');
|
||||
expect(custom()).toBe(true);
|
||||
write('profile = "missing"\n');
|
||||
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
|
||||
write('model_provider = 7\n');
|
||||
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
|
||||
write('model_provider = [\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('native_provider_unresolved');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toBe('gpt-x');
|
||||
write('openai_base_url = "https://gateway.example/v1"\n');
|
||||
expect(custom()).toBe(true);
|
||||
write('openai_base_url = "https://api.openai.com/v1/"\n');
|
||||
expect(custom()).toBe(false);
|
||||
expect(detectCustomProvider('openai', { env: { ...env, OPENAI_BASE_URL: 'https://gateway.example/v1' } }).custom).toBe(true);
|
||||
expect(policyError(() => detectCustomProvider('openai', { env: { ...env, CODEX_HOME: 'relative/codex' } })).reason).toBe('native_provider_unresolved');
|
||||
const alt = path.join(home, 'codex home with spaces');
|
||||
fs.mkdirSync(alt);
|
||||
fs.writeFileSync(path.join(alt, 'config.toml'), 'model_provider = "azure"\n');
|
||||
expect(detectCustomProvider('openai', { env: { ...env, CODEX_HOME: alt } }).custom).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('custom-provider settings scopes', () => {
|
||||
function project(settings: Record<string, string>) {
|
||||
const fx = fixture();
|
||||
for (const [rel, body] of Object.entries(settings)) {
|
||||
fs.mkdirSync(path.dirname(path.join(fx.root, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(fx.root, rel), body);
|
||||
}
|
||||
return fx;
|
||||
}
|
||||
const proxy = JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } });
|
||||
const bedrock = JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } });
|
||||
|
||||
test.each([
|
||||
['shared project settings', 'app/.claude/settings.json', 'app'],
|
||||
['project local settings', 'app/.claude/settings.local.json', 'app'],
|
||||
['repository-root local settings from a subdirectory', 'repo/.claude/settings.local.json', 'repo/pkg'],
|
||||
])('%s block the public default', (_name, file, cwd) => {
|
||||
const fx = project({ [file]: proxy, 'repo/.git/HEAD': 'ref: refs/heads/main\n' });
|
||||
fs.mkdirSync(path.join(fx.root, cwd), { recursive: true });
|
||||
const scope = { env: fx.env, cwd: path.join(fx.root, cwd) };
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', ...scope }));
|
||||
expect(error.reason).toBe('custom_provider_requires_model');
|
||||
expect(error.cause).toContain(path.join(fx.root, file));
|
||||
expect(error.cause).not.toContain('proxy.example.test');
|
||||
expect(policyError(() => resolveImplementationModels({ host: 'claude-code', ...scope })).reason).toBe('custom_provider_requires_model');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_CLAUDE_MODEL: 'proxy-claude' } }))).toBe('proxy-claude');
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_STATE_ROOT: project({ 'state/config.yaml': 'plan_review_tier: host\n' }).state } }).status).toBe('delegated-host');
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', ...scope }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('a worktree also reads the main checkout local settings', () => {
|
||||
const fx = project({
|
||||
'main/.git/worktrees/wt/commondir': '../..\n',
|
||||
'main/.claude/settings.local.json': bedrock,
|
||||
});
|
||||
fs.mkdirSync(path.join(fx.root, 'wt'));
|
||||
fs.writeFileSync(path.join(fx.root, 'wt', '.git'), `gitdir: ${path.join(fx.root, 'main/.git/worktrees/wt')}\n`);
|
||||
const detection = detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'wt') });
|
||||
expect(detection.signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(fx.root, 'main/.claude/settings.local.json')})`]);
|
||||
});
|
||||
|
||||
test('shared settings are read from the working directory, as Claude does, not the repository root', () => {
|
||||
const fx = project({ 'repo/.git/HEAD': '', 'repo/.claude/settings.json': proxy });
|
||||
fs.mkdirSync(path.join(fx.root, 'repo/pkg'));
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo/pkg') }).custom).toBe(false);
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo') }).custom).toBe(true);
|
||||
});
|
||||
|
||||
test('managed settings file and drop-ins count; a malformed one is unresolved', () => {
|
||||
const fx = project({ 'managed/managed-settings.d/20-route.json': bedrock, 'managed/managed-settings.d/.hidden.json': '{', 'managed/managed-settings.d/notes.txt': '{' });
|
||||
const managed = path.join(fx.root, 'managed');
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, claudeManagedDir: managed }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(managed, 'managed-settings.d/20-route.json')})`]);
|
||||
fs.writeFileSync(path.join(managed, 'managed-settings.json'), '{broken');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: fx.env, claudeManagedDir: managed }))).toMatchObject({ reason: 'native_provider_unresolved', key: path.join(managed, 'managed-settings.json') });
|
||||
});
|
||||
|
||||
test('a requested cwd is honored instead of process.cwd()', () => {
|
||||
const fx = project({ 'elsewhere/.claude/settings.local.json': proxy });
|
||||
const cwd = path.join(fx.root, 'elsewhere');
|
||||
expect(process.cwd()).not.toBe(cwd);
|
||||
const managed = { claudeManagedDir: path.join(fx.root, 'none') };
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd, ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd: path.relative(process.cwd(), cwd), ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
const original = process.cwd();
|
||||
process.chdir(fx.home);
|
||||
try {
|
||||
expect(planReviewWith({ provider: 'anthropic', env: fx.env, ...managed }).source.kind).toBe('catalog');
|
||||
process.chdir(cwd);
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
} finally {
|
||||
process.chdir(original);
|
||||
}
|
||||
});
|
||||
|
||||
test('Codex system config is a lower layer than user config', () => {
|
||||
const fx = project({ 'etc/codex.toml': 'model_provider = "azure"\n' });
|
||||
const scope = { env: fx.env, codexSystemConfig: path.join(fx.root, 'etc/codex.toml') };
|
||||
expect(detectCustomProvider('openai', scope).signals).toEqual([`model_provider "azure" (${scope.codexSystemConfig})`]);
|
||||
fs.mkdirSync(path.join(fx.home, '.codex'));
|
||||
fs.writeFileSync(path.join(fx.home, '.codex', 'config.toml'), 'model_provider = "openai"\n');
|
||||
expect(detectCustomProvider('openai', scope).custom).toBe(false);
|
||||
});
|
||||
|
||||
test('a malformed native config reached from smart plan-review or implementation names no tier override', () => {
|
||||
const fx = project({ 'state/config.yaml': 'plan_review_tier: smart\n', 'home/.codex/config.toml': 'model_provider = [\n' });
|
||||
for (const run of [
|
||||
() => resolvePlanReviewModel({ provider: 'openai', env: fx.env }),
|
||||
() => resolveImplementationModels({ provider: 'openai', env: fx.env }),
|
||||
]) {
|
||||
const error = policyError(run);
|
||||
expect(error.reason).toBe('native_provider_unresolved');
|
||||
expect(error.repair).toEqual([`fix the TOML in ${path.join(fx.home, '.codex', 'config.toml')}`, 'or name the model explicitly for this request']);
|
||||
}
|
||||
});
|
||||
|
||||
test('detection reports what it scanned and what it cannot see', () => {
|
||||
const fx = fixture();
|
||||
const detection = detectCustomProvider('anthropic', { env: fx.env });
|
||||
expect(detection.scanned).toContain(path.join(ISOLATED_DIR, '.claude', 'settings.local.json'));
|
||||
expect(detection.scanned).toContain(path.join(ISOLATED.claudeManagedDir, 'managed-settings.json'));
|
||||
expect(detection.unscanned).toEqual([...CUSTOM_PROVIDER_UNSCANNED.anthropic]);
|
||||
expect(detectCustomProvider('openai', { env: fx.env }).unscanned.join(' ')).toContain('--profile');
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation recommendation', () => {
|
||||
test('known hosts recommend their own provider; unknown hosts get both', () => {
|
||||
const { env } = fixture();
|
||||
expect(resolveImplementationModels({ host: 'claude-code', env }).map(s => [s.provider, selected(s)])).toEqual([['anthropic', 'claude-opus-5-5']]);
|
||||
expect(resolveImplementationModels({ host: 'codex', env }).map(s => [s.provider, selected(s)])).toEqual([['openai', 'gpt-6.1-sol']]);
|
||||
for (const host of ['unknown', undefined] as const) {
|
||||
expect(resolveImplementationModels({ host, env }).map(s => s.provider)).toEqual(['anthropic', 'openai']);
|
||||
}
|
||||
expect(resolveImplementationModels({ host: 'codex', provider: 'anthropic', env }).map(s => s.provider)).toEqual(['anthropic']);
|
||||
});
|
||||
|
||||
test('frontier is honored, overrides apply, env overrides do not', () => {
|
||||
const { env } = fixture('implementation_tier: frontier\nmodel_frontier_openai: gpt-pin\n');
|
||||
const envWithOverrides = { ...env, GSTACK_CLAUDE_MODEL: 'claude-env', GSTACK_CODEX_MODEL: 'gpt-env' };
|
||||
const [anthropic, openai] = resolveImplementationModels({ env: envWithOverrides });
|
||||
expect(anthropic).toMatchObject({ role: 'implementation', tier: 'frontier', requestedModel: 'claude-fable-5-1', source: { kind: 'catalog' }, tierFrom: { key: 'implementation_tier', origin: 'config' } });
|
||||
expect(openai).toMatchObject({ requestedModel: 'gpt-pin', source: { kind: 'config', key: 'model_frontier_openai' } });
|
||||
expect(resolveImplementationModels({ provider: 'openai', requestedModel: 'gpt-req', env })[0]).toMatchObject({ requestedModel: 'gpt-req', source: { kind: 'request' } });
|
||||
});
|
||||
|
||||
test('an explicit model without a single provider is rejected', () => {
|
||||
const { env } = fixture();
|
||||
expect(policyError(() => resolveImplementationModels({ requestedModel: 'gpt-x', env })).reason).toBe('provider_required');
|
||||
});
|
||||
|
||||
test('custom routing needs an explicit implementation model; it is never delegated', () => {
|
||||
const { env } = fixture();
|
||||
const custom = { ...env, CLAUDE_CODE_USE_BEDROCK: '1' };
|
||||
const error = policyError(() => resolveImplementationModels({ host: 'claude-code', env: custom }));
|
||||
expect(error).toMatchObject({ reason: 'custom_provider_requires_model' });
|
||||
expect(error.repair[0]).toBe('gstack-config set model_smart_claude <model-id-your-endpoint-serves>');
|
||||
expect(error.repair.join(' ')).not.toContain('GSTACK_CLAUDE_MODEL');
|
||||
expect(error.repair.join(' ')).not.toContain('host');
|
||||
expect(selected(resolveImplementationModels({ host: 'claude-code', env: { ...fixture('model_smart_claude: us.anthropic.claude-x\n').env, CLAUDE_CODE_USE_BEDROCK: '1' } })[0])).toBe('us.anthropic.claude-x');
|
||||
});
|
||||
});
|
||||
|
||||
describe('purity', () => {
|
||||
test('resolution and inspection never write state', () => {
|
||||
const { root, env } = fixture('plan_review_tier: smart\n');
|
||||
const before = tree(root);
|
||||
resolvePlanReviewModel({ provider: 'openai', env });
|
||||
resolvePlanReviewModel({ provider: 'anthropic', env });
|
||||
resolveImplementationModels({ env });
|
||||
expect(tree(root)).toEqual(before);
|
||||
});
|
||||
|
||||
test('errors carry every structured field', () => {
|
||||
const error = policyError(() => parseModelPolicyConfig('plan_review_tier: nope\n', '/c.yaml'));
|
||||
expect(Object.keys(error.toJSON()).sort()).toEqual(['cause', 'docs', 'key', 'problem', 'reason', 'repair', 'source']);
|
||||
expect(error.repair.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,485 @@
|
||||
/**
|
||||
* Role-bearing outside invocations: exact plan-review membership, one bound
|
||||
* selection through probe and dispatch, the invocation-boundary notice, and
|
||||
* no-role negative controls. Fake CLIs only; no paid calls.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { outsideVoiceCommand, outsideVoicePreflight } from '../scripts/resolvers/outside-voice';
|
||||
import { RESOLVERS } from '../scripts/resolvers/index';
|
||||
import { generateAdversarialStep, generateCodexDocReview, generateCodexPlanReview, generateCodexSecondOpinion } from '../scripts/resolvers/outside-voice-steps';
|
||||
import { generateDesignOutsideVoices } from '../scripts/resolvers/design';
|
||||
import { generateImplementationModelHandoff } from '../scripts/resolvers/plan-review';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
import { claudeCodeArgs } from '../lib/claude-code';
|
||||
import { resolvePlanReviewModel } from '../lib/model-policy';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'role-invocation-'));
|
||||
const REPO = path.join(TMP, 'repo');
|
||||
const BIN = path.join(TMP, 'bin');
|
||||
const CALLS = path.join(TMP, 'calls.jsonl');
|
||||
const PROMPT = path.join(TMP, 'prompt.txt');
|
||||
const FAKE = path.join(TMP, 'fake.ts');
|
||||
for (const dir of [REPO, BIN]) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(PROMPT, 'Review this plan.\n');
|
||||
fs.writeFileSync(FAKE, `
|
||||
import { appendFileSync, writeFileSync } from 'node:fs';
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === 'sandbox') process.exit(0);
|
||||
if (args[0] === '--version') { console.log('codex-cli ' + (process.env.FAKE_CODEX_VERSION ?? '0.160.0')); process.exit(0); }
|
||||
const claude = process.env.FAKE_AS === 'claude';
|
||||
const probe = !claude && args.includes('reply OK');
|
||||
appendFileSync(process.env.CALLS!, JSON.stringify({ args, probe }) + '\\n');
|
||||
if (probe) {
|
||||
if (process.env.REPLACE_CONFIG) writeFileSync(process.env.GSTACK_HOME + '/config.yaml', process.env.REPLACE_CONFIG);
|
||||
if (process.env.FAKE_PROBE === 'model400') { console.error('ERROR: {"status":400,"error":{"message":"The model is not supported."}}'); process.exit(1); }
|
||||
if (process.env.FAKE_PROBE === 'broken') { console.error('Error: spawn codex-vendor ENOENT'); process.exit(1); }
|
||||
console.log('OK');
|
||||
process.exit(0);
|
||||
}
|
||||
if (!claude || args.includes('-')) await Bun.stdin.text();
|
||||
const response = 'Recommendation: split the migration because the plan couples two rollouts.';
|
||||
if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } }));
|
||||
else writeFileSync(args[args.indexOf('-o') + 1], response);
|
||||
`);
|
||||
fs.writeFileSync(path.join(BIN, 'codex'), `#!/usr/bin/env bash\nexec bun "${FAKE}" "$@"\n`, { mode: 0o755 });
|
||||
|
||||
const git = (args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: REPO, encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@example.invalid', GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@example.invalid' } });
|
||||
if (r.status !== 0) throw new Error(r.stderr);
|
||||
};
|
||||
git(['init', '-q', '-b', 'main']);
|
||||
fs.writeFileSync(path.join(REPO, 'plan.md'), 'plan\n');
|
||||
git(['add', 'plan.md']);
|
||||
git(['commit', '-qm', 'init']);
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
type Host = 'claude' | 'codex';
|
||||
const ctxFor = (skillName: string, host: Host, paths = HOST_PATHS.codex): TemplateContext =>
|
||||
({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, host, paths });
|
||||
|
||||
let counter = 0;
|
||||
interface Run { stdout: string; stderr: string; status: number | null; calls: Array<{ args: string[]; probe: boolean }>; state: string }
|
||||
|
||||
function home(config = '', codexToml = 'model = "gpt-5.6-terra"\n') {
|
||||
const dir = path.join(TMP, `h${++counter}`);
|
||||
const state = path.join(dir, 'state');
|
||||
fs.mkdirSync(path.join(dir, '.codex'), { recursive: true });
|
||||
fs.mkdirSync(state, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, '.codex', 'config.toml'), codexToml);
|
||||
if (config) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
return { dir, state };
|
||||
}
|
||||
|
||||
function env(h: { dir: string; state: string }, host: Host, extra: Record<string, string> = {}): Record<string, string> {
|
||||
return {
|
||||
PATH: `${BIN}${path.delimiter}${process.env.PATH ?? ''}`, HOME: h.dir, CODEX_HOME: path.join(h.dir, '.codex'),
|
||||
GSTACK_HOME: h.state, GSTACK_ROOT: ROOT, GSTACK_BIN: path.join(ROOT, 'bin'), CALLS, CODEX_API_KEY: 'test-key',
|
||||
GSTACK_CLAUDE_BIN: process.execPath, GSTACK_CLAUDE_BIN_ARGS: JSON.stringify([FAKE]), FAKE_AS: host === 'codex' ? 'claude' : 'codex',
|
||||
...(host === 'codex' ? { CODEX_THREAD_ID: 'fixture', GSTACK_ACTIVE_HOST: 'codex' } : { CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' }),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function run(host: Host, h: { dir: string; state: string }, extra: Record<string, string> = {}, role: 'plan-review' | null = 'plan-review'): Run {
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const command = outsideVoiceCommand(ctxFor('plan-eng-review', host), { promptFile: PROMPT, timeoutMs: 8000, ...(role ? { role } : {}) });
|
||||
const r = spawnSync('bash', ['-c', command], { cwd: REPO, env: env(h, host, extra), encoding: 'utf8', timeout: 30000 });
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
||||
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status, calls, state: h.state };
|
||||
}
|
||||
|
||||
const codexModel = (call: { args: string[] }) => call.args.find(arg => arg.startsWith('model='));
|
||||
const claudeModels = (call: { args: string[] }) => call.args.flatMap((arg, i) => (arg === '--model' ? [call.args[i + 1]] : arg.startsWith('--model=') ? [arg.slice(8)] : []));
|
||||
const NOTICE = 'NOTICE: gstack plan reviews now use an independent plan-review model';
|
||||
const marker = (state: string) => path.join(state, '.model-policy-notice-v1');
|
||||
|
||||
describe('plan-review role membership in shared generators', () => {
|
||||
const roleBearing: Array<[string, (ctx: TemplateContext) => string]> = [
|
||||
['autoplan', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan'])],
|
||||
['spec', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['spec'])],
|
||||
['plan-ceo-review', generateCodexPlanReview],
|
||||
['plan-eng-review', generateCodexPlanReview],
|
||||
['plan-devex-review', generateCodexPlanReview],
|
||||
['plan-design-review', generateDesignOutsideVoices],
|
||||
];
|
||||
const noRole: Array<[string, (ctx: TemplateContext) => string]> = [
|
||||
['design-review', generateDesignOutsideVoices],
|
||||
['design-consultation', generateDesignOutsideVoices],
|
||||
['office-hours', generateCodexSecondOpinion],
|
||||
['office-hours', ctx => RESOLVERS.DESIGN_SKETCH(ctx)],
|
||||
['review', generateAdversarialStep],
|
||||
['ship', generateAdversarialStep],
|
||||
['ship', ctx => RESOLVERS.DESIGN_REVIEW_LITE(ctx)],
|
||||
['document-release', generateCodexDocReview],
|
||||
];
|
||||
for (const host of ['claude', 'codex'] as const) {
|
||||
const roleMarker = host === 'claude' ? '_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?' : '--role plan-review';
|
||||
for (const [skill, render] of roleBearing) {
|
||||
test(`${host}: ${skill} invokes with the plan-review role only`, () => {
|
||||
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
||||
expect(text).toContain(roleMarker);
|
||||
expect(text).not.toMatch(/_gstack_codex_select_model (exec|review) \|\| exit 1/);
|
||||
expect(text).not.toContain('_gstack_codex_model_probe');
|
||||
expect(text).not.toContain('_CODEX_MP');
|
||||
expect(text).not.toContain('without overriding either');
|
||||
});
|
||||
}
|
||||
for (const [skill, render] of noRole) {
|
||||
test(`${host}: ${skill} stays no-role`, () => {
|
||||
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
||||
expect(text).not.toContain('plan-review');
|
||||
expect(text).not.toContain('--role');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test('spec keeps medium effort, its 120s deadline and spec gate on both providers', () => {
|
||||
const codex = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'claude'), ['spec']);
|
||||
expect(codex).toContain(`model_reasoning_effort="medium"`);
|
||||
expect(codex).toContain('_gstack_codex_timeout_wrapper 120 codex exec');
|
||||
expect(codex).toMatch(/ spec "\$_OUTSIDE_TMP\/text"/);
|
||||
const claude = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'codex'), ['spec']);
|
||||
expect(claude).toContain('--timeout-ms 120000 --role plan-review');
|
||||
});
|
||||
|
||||
test('role-bearing availability checks pay for no model probe; no-role checks keep theirs', () => {
|
||||
for (const host of ['claude', 'codex'] as const) {
|
||||
for (const text of [
|
||||
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('autoplan', host, HOST_PATHS.claude), ['autoplan']),
|
||||
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', host, HOST_PATHS.claude), ['opt-in', 'plan-review']),
|
||||
outsideVoicePreflight(ctxFor('plan-eng-review', host, HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' }),
|
||||
]) {
|
||||
expect(text).not.toMatch(/_gstack_codex_(model|auth)_probe|_CODEX_MP|without overriding either/);
|
||||
expect(text).toContain('the plan-review model (policy-selected, printed with its source) are checked by the actual invocation');
|
||||
}
|
||||
}
|
||||
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).toContain('without overriding either');
|
||||
expect(generateCodexDocReview(ctxFor('document-release', 'claude', HOST_PATHS.claude))).toContain('{ _gstack_codex_model_probe; _CODEX_MP=$?; }');
|
||||
expect(generateAdversarialStep(ctxFor('review', 'claude', HOST_PATHS.claude))).toContain('_gstack_codex_model_probe review');
|
||||
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).not.toContain('_gstack_codex_model_probe');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex plan-review invocation binds one selection', () => {
|
||||
test('a known-bad CLI version still warns without changing its advisory policy', () => {
|
||||
const result = run('claude', home(), { FAKE_CODEX_VERSION: '0.120.2' });
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toMatch(/WARN:.*0\.120\.2.*known stdin deadlock/);
|
||||
expect(result.calls.filter(call => call.probe)).toHaveLength(1);
|
||||
expect(result.calls.filter(call => !call.probe)).toHaveLength(1);
|
||||
expect(result.calls.every(call => codexModel(call) === 'model="gpt-6-astra"')).toBe(true);
|
||||
});
|
||||
|
||||
test('frontier catalog model wins over a native pin; probe and dispatch use it; notice once', () => {
|
||||
const h = home();
|
||||
const first = run('claude', h);
|
||||
expect(first.status).toBe(0);
|
||||
expect(first.stdout).toContain('OUTSIDE_STATUS: completed provider=codex');
|
||||
expect(first.calls.map(c => c.probe)).toEqual([true, false]);
|
||||
expect(first.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
||||
expect(first.stderr).toContain('CODEX_MODEL: gpt-6-astra (exec; role: plan-review, tier: frontier; source: gstack catalog frontier/openai');
|
||||
expect(first.stderr).toContain(NOTICE);
|
||||
expect(first.stderr).toContain('gstack-config set plan_review_tier smart');
|
||||
expect(first.stderr.indexOf(NOTICE)).toBeLessThan(first.stderr.indexOf('CODEX_MODEL:'));
|
||||
expect(fs.existsSync(marker(h.state))).toBe(true);
|
||||
const second = run('claude', h);
|
||||
expect(second.status).toBe(0);
|
||||
expect(second.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(second.stdout).toContain('MODEL_OK (cached)');
|
||||
expect(second.calls.map(c => c.probe)).toEqual([false]);
|
||||
});
|
||||
|
||||
test('negative control: the same call without a role keeps native selection, no probe, no notice', () => {
|
||||
const h = home();
|
||||
const r = run('claude', h, {}, null);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([false]);
|
||||
expect(r.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
});
|
||||
|
||||
test('a config replaced between probe and dispatch affects only the next invocation', () => {
|
||||
const h = home();
|
||||
const r = run('claude', h, { REPLACE_CONFIG: 'model_frontier_openai: gpt-replaced\n' });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
||||
const next = run('claude', h);
|
||||
expect(next.calls.map(codexModel)).toEqual(['model="gpt-replaced"', 'model="gpt-replaced"']);
|
||||
expect(next.stderr).toContain('source: model_frontier_openai in');
|
||||
});
|
||||
|
||||
for (const [label, config, extra, model, fragment] of [
|
||||
['smart tier', 'plan_review_tier: smart\n', {}, 'gpt-6.1-sol', 'tier: smart'],
|
||||
['env override', '', { GSTACK_CODEX_MODEL: 'gpt-env-pick' }, 'gpt-env-pick', 'source: GSTACK_CODEX_MODEL'],
|
||||
['host mode', 'plan_review_tier: host\n', {}, 'gpt-5.6-terra', 'source: plan_review_tier=host'],
|
||||
] as const) {
|
||||
test(`${label} reaches both probe and dispatch with its real source`, () => {
|
||||
const r = run('claude', home(config), extra);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(codexModel)).toEqual([`model="${model}"`, `model="${model}"`]);
|
||||
expect(r.stderr).toContain(fragment);
|
||||
});
|
||||
}
|
||||
|
||||
test('invalid policy config and unrouted custom providers stop before any Codex process', () => {
|
||||
const invalid = run('claude', home('plan_review_tier: turbo\n'));
|
||||
expect(invalid.status).toBe(1);
|
||||
expect(invalid.calls).toEqual([]);
|
||||
expect(invalid.stderr).toContain('gstack-config unset plan_review_tier');
|
||||
expect(invalid.stderr).toContain('outside review unavailable; missing coverage');
|
||||
const custom = run('claude', home('', 'model_provider = "azure"\n'));
|
||||
expect(custom.status).toBe(1);
|
||||
expect(custom.calls).toEqual([]);
|
||||
expect(custom.stderr).toContain('model_frontier_openai');
|
||||
expect(custom.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
test('a rejected role model is unusable with a source-aware repair and never dispatches', () => {
|
||||
const r = run('claude', home(), { FAKE_PROBE: 'model400' });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
||||
expect(r.stdout).toContain('MODEL_UNUSABLE');
|
||||
expect(r.stdout).toContain('source: gstack catalog frontier/openai');
|
||||
expect(r.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
|
||||
expect(r.stdout).not.toContain('set model in');
|
||||
});
|
||||
|
||||
test('a broken CLI found by the role probe exits 2 and never dispatches', () => {
|
||||
const r = run('claude', home(), { FAKE_PROBE: 'broken' });
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
||||
expect(r.stdout).toContain('MODEL_UNUSABLE_INSTALL');
|
||||
expect(r.stdout).toContain('npm install -g @openai/codex');
|
||||
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('missing authentication makes no Codex call', () => {
|
||||
const r = run('claude', home(), { CODEX_API_KEY: '' });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(r.stdout).toContain('AUTH_FAILED');
|
||||
});
|
||||
|
||||
test('an explicit request outranks GSTACK_CODEX_MODEL in the probe helper', () => {
|
||||
const h = home();
|
||||
const r = spawnSync('bash', ['-c', `source "${ROOT}/bin/gstack-codex-probe" && _gstack_codex_select_model exec gpt-request plan-review && echo "SEL=$_GSTACK_CODEX_SEL"`],
|
||||
{ env: env(h, 'claude', { GSTACK_CODEX_MODEL: 'gpt-env' }), encoding: 'utf8', timeout: 15000 });
|
||||
expect(r.stdout).toContain('SEL=gpt-request');
|
||||
expect(r.stderr).toContain('source: explicit request');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Claude plan-review invocation emits one selected model or none', () => {
|
||||
const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!);
|
||||
|
||||
test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => {
|
||||
const h = home();
|
||||
const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls).toHaveLength(1);
|
||||
expect(claudeModels(r.calls[0])).toEqual(['claude-fable-5-1']);
|
||||
expect(result(r).selection).toMatchObject({ role: 'plan-review', tier: 'frontier', status: 'selected', requested_model: 'claude-fable-5-1' });
|
||||
expect(result(r).modelUsage).toHaveProperty('model-actual');
|
||||
expect(r.stderr).toContain('CLAUDE_MODEL: plan-review via anthropic: claude-fable-5-1');
|
||||
expect(r.stderr).toContain(NOTICE);
|
||||
expect(run('codex', h).stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
test('GSTACK_CLAUDE_MODEL ranks inside the record and appears once', () => {
|
||||
const r = run('codex', home(), { GSTACK_CLAUDE_MODEL: 'claude-env-pick' });
|
||||
expect(claudeModels(r.calls[0])).toEqual(['claude-env-pick']);
|
||||
expect(result(r).selection.source).toContain('GSTACK_CLAUDE_MODEL');
|
||||
});
|
||||
|
||||
test('host mode delegates: zero --model flags and an unknown requested model', () => {
|
||||
const r = run('codex', home('plan_review_tier: host\n'));
|
||||
expect(r.status).toBe(0);
|
||||
expect(claudeModels(r.calls[0])).toEqual([]);
|
||||
expect(result(r).selection).toMatchObject({ status: 'delegated-host', requested_model: null });
|
||||
});
|
||||
|
||||
test('custom Anthropic routing without an explicit model never spawns Claude', () => {
|
||||
const r = run('codex', home(), { ANTHROPIC_BASE_URL: 'https://proxy.example.invalid' });
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(r.stdout).toContain('custom_provider_requires_model');
|
||||
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('target-repo project settings with a custom endpoint stop a catalog default, whatever the parent cwd', () => {
|
||||
const target = path.join(TMP, 'custom-target');
|
||||
fs.mkdirSync(path.join(target, '.claude'), { recursive: true });
|
||||
fs.writeFileSync(path.join(target, '.claude', 'settings.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://gateway.example.invalid' } }));
|
||||
const sub = path.join(target, 'pkg');
|
||||
fs.mkdirSync(sub, { recursive: true });
|
||||
spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: target, timeout: 5000 });
|
||||
const h = home();
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const direct = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
||||
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
||||
expect(direct.status).toBe(1);
|
||||
expect(JSON.parse(direct.stdout).error.code).toBe('custom_provider_requires_model');
|
||||
expect(fs.existsSync(CALLS)).toBe(false);
|
||||
const command = outsideVoiceCommand(ctxFor('plan-eng-review', 'codex'), { promptFile: PROMPT, timeoutMs: 5000, role: 'plan-review' });
|
||||
const generated = spawnSync('bash', ['-c', command], { cwd: sub, env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
||||
expect(generated.status).not.toBe(0);
|
||||
expect(generated.stdout).toContain('custom_provider_requires_model');
|
||||
expect(fs.existsSync(CALLS)).toBe(false);
|
||||
const explicit = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
||||
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex', { GSTACK_CLAUDE_MODEL: 'gateway-model' }), encoding: 'utf8', timeout: 20000 });
|
||||
expect(explicit.status).toBe(0);
|
||||
expect(JSON.parse(explicit.stdout).selection.requested_model).toBe('gateway-model');
|
||||
});
|
||||
|
||||
test('negative control: no role keeps zero --model without an override and shows no notice', () => {
|
||||
const h = home();
|
||||
const r = run('codex', h, {}, null);
|
||||
expect(r.status).toBe(0);
|
||||
expect(claudeModels(r.calls[0])).toEqual([]);
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
});
|
||||
|
||||
test('a selected record is never re-resolved from the environment', () => {
|
||||
const cmd = { command: 'claude', argsPrefix: [] };
|
||||
const h = home();
|
||||
const selected = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
||||
if (selected.status !== 'selected') throw new Error('expected a selected record');
|
||||
const args = claudeCodeArgs({ access: 'none', selection: selected }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' });
|
||||
expect(claudeModels({ args })).toEqual(['claude-fable-5-1']);
|
||||
const dashed = claudeCodeArgs({ access: 'none', selection: { ...selected, requestedModel: '-p' } }, cmd, {});
|
||||
expect(dashed.filter(arg => arg === '-p')).toHaveLength(1);
|
||||
expect(dashed).toContain('--model=-p');
|
||||
fs.writeFileSync(path.join(h.state, 'config.yaml'), 'plan_review_tier: host\n');
|
||||
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
||||
expect(claudeModels({ args: claudeCodeArgs({ access: 'none', selection: delegated }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' }) })).toEqual([]);
|
||||
expect(claudeCodeArgs({ access: 'none' }, cmd, { GSTACK_CLAUDE_MODEL: 'legacy pick' }).slice(-2)).toEqual(['--model', 'legacy pick']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('manual /codex entry: explicit role only, paid probe only for the dispatched model', () => {
|
||||
const skill = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md.tmpl'), 'utf8');
|
||||
const preflight = skill.match(/```bash\n(_TEL=[^\n]*\n_CODEX_ROLE=''[\s\S]*?)\n```/)![1]!
|
||||
.replace('{{OUTSIDE_SELF_GUARD:codex}}', RESOLVERS.OUTSIDE_SELF_GUARD(ctxFor('codex', 'claude', HOST_PATHS.claude), ['codex']));
|
||||
const withRole = (text: string, role: string) => text.replaceAll("_CODEX_ROLE=''", `_CODEX_ROLE='${role}'`);
|
||||
const shell = (h: { dir: string; state: string }, script: string) => {
|
||||
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
||||
if (!fs.existsSync(path.join(h.dir, '.claude', 'skills', 'gstack'))) fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const r = spawnSync('bash', ['-c', script], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 30000 });
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
||||
return { ...r, calls: calls.filter((c: { args: string[] }) => c.args[0] === 'exec') };
|
||||
};
|
||||
|
||||
test('Step 0.5 probes the native model without a role and makes no model call with it', () => {
|
||||
const plain = shell(home(), preflight);
|
||||
expect(plain.status).toBe(0);
|
||||
expect(plain.stdout).toContain('MODEL_OK');
|
||||
expect(plain.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
||||
const role = shell(home(), withRole(preflight, 'plan-review'));
|
||||
expect(role.status).toBe(0);
|
||||
expect(role.stdout).not.toMatch(/MODEL_OK|MODEL_UNUSABLE|AUTH_FAILED/);
|
||||
expect(role.calls).toEqual([]);
|
||||
expect(role.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) {
|
||||
const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8');
|
||||
const selections = [...template.matchAll(/_CODEX_ROLE=''\n(_gstack_codex_select_model (exec|review) '' "\$_CODEX_ROLE" "\$_REPO_ROOT" \|\| exit 1\n\[ -z "\$_CODEX_ROLE" \] \|\| _gstack_codex_model_probe \2 \|\| exit \$\?)\n/g)];
|
||||
test(`${file}: every dispatch block selects then, with the role only, probes that same model`, () => {
|
||||
expect(selections.length).toBe(file === 'review-mode' ? 2 : 1);
|
||||
expect(template.match(/_gstack_codex_select_model/g)!.length).toBe(selections.length + (file === 'review-mode' ? 1 : 0));
|
||||
for (const [block, , kind] of selections) {
|
||||
const script = (role: string) => `_REPO_ROOT='${REPO}'\nsource "${ROOT}/bin/gstack-codex-probe" || exit 1\n${withRole(block, role)}\necho "DISPATCH=$_GSTACK_CODEX_SEL"`;
|
||||
const plain = shell(home(), script(''));
|
||||
expect(plain.status).toBe(0);
|
||||
expect(plain.stdout).toContain('DISPATCH=gpt-5.6-terra');
|
||||
expect(plain.calls).toEqual([]);
|
||||
const h = home();
|
||||
const role = shell(h, script('plan-review'));
|
||||
expect(role.status).toBe(0);
|
||||
expect(role.stdout).toContain('DISPATCH=gpt-6-astra');
|
||||
expect(role.calls.map(codexModel)).toEqual(['model="gpt-6-astra"']);
|
||||
expect(role.stderr).toContain(`CODEX_MODEL: gpt-6-astra (${kind}; role: plan-review, tier: frontier`);
|
||||
expect(role.stderr).toContain(NOTICE);
|
||||
const rejected = shell(home(), `export FAKE_PROBE=model400\n${script('plan-review')}`);
|
||||
expect(rejected.status).toBe(1);
|
||||
expect(rejected.stdout).not.toContain('DISPATCH=');
|
||||
expect(rejected.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('invocation-boundary migration notice', () => {
|
||||
test('pure inspection writes nothing and does not suppress the first real notice', () => {
|
||||
const h = home();
|
||||
const inspect = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'plan-review', '--provider', 'openai', '--json'],
|
||||
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
||||
expect(inspect.status).toBe(0);
|
||||
expect(fs.readdirSync(h.state)).toEqual([]);
|
||||
expect(run('claude', h).stderr).toContain(NOTICE);
|
||||
});
|
||||
|
||||
for (const host of ['codex', 'claude'] as const) {
|
||||
test(`${host === 'codex' ? 'Claude' : 'Codex'}: an unwritable marker repeats the notice instead of hiding it`, () => {
|
||||
const h = home();
|
||||
fs.chmodSync(h.state, 0o500);
|
||||
try {
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const r = run(host, h);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain(NOTICE);
|
||||
}
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
} finally { fs.chmodSync(h.state, 0o700); }
|
||||
});
|
||||
}
|
||||
|
||||
test('disabled reviews stay disabled: no probe, no notice, no Codex call', () => {
|
||||
const block = outsideVoicePreflight(ctxFor('plan-eng-review', 'claude', HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' })
|
||||
.match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
||||
for (const [config, mode] of [['codex_reviews: disabled\n', 'CODEX_MODE: disabled'], ['', 'CODEX_MODE: ready']] as const) {
|
||||
const h = home(config);
|
||||
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
||||
fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const r = spawnSync('bash', ['-c', block], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 20000 });
|
||||
expect(r.stdout).toContain(mode);
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8') : '';
|
||||
expect(calls).not.toContain('"exec"');
|
||||
if (config) expect(calls).toBe('');
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation handoff recommendation', () => {
|
||||
test('known harnesses pass their native provider; other hosts get both; nothing switches the session', () => {
|
||||
expect(generateImplementationModelHandoff(ctxFor('autoplan', 'claude', HOST_PATHS.claude))).toContain('gstack-models" resolve --role implementation --provider anthropic`');
|
||||
expect(generateImplementationModelHandoff(ctxFor('spec', 'codex'))).toContain('resolve --role implementation --provider openai`');
|
||||
const other = generateImplementationModelHandoff(ctxFor('plan-eng-review', 'kiro' as Host));
|
||||
expect(other).toContain('resolve --role implementation`');
|
||||
expect(other).toContain('one per provider');
|
||||
for (const text of [other]) {
|
||||
expect(text).toContain('cannot change this session');
|
||||
expect(text).not.toMatch(/claude-|gpt-/);
|
||||
}
|
||||
});
|
||||
|
||||
test('the recommended command resolves offline from current settings without writes', () => {
|
||||
const h = home('implementation_tier: frontier\n');
|
||||
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'implementation', '--provider', 'anthropic', '--json'],
|
||||
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
||||
expect(r.status).toBe(0);
|
||||
expect(JSON.parse(r.stdout).selections[0]).toMatchObject({ role: 'implementation', requestedModel: 'claude-fable-5-1' });
|
||||
expect(fs.readdirSync(h.state)).toEqual(['config.yaml']);
|
||||
});
|
||||
});
|
||||
@@ -114,7 +114,7 @@ describe('own-harness review fallback instructions', () => {
|
||||
test(`${host.name}: ${name} fallback names only the mode its preflight emits`, () => {
|
||||
const ctx: TemplateContext = { host: host.name, skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', paths: HOST_PATHS[host.name] };
|
||||
const text = render(ctx);
|
||||
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const mode = name === 'plan' || host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const preflight = text.match(/```bash\n([\s\S]*?)\n```/)![1];
|
||||
expect(preflight).toContain(mode);
|
||||
expect([...new Set(text.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
|
||||
|
||||
@@ -138,8 +138,7 @@ describe('CI workflow clarity regressions', () => {
|
||||
test('CEO fallback names the current host mode and needs completed findings before the later report exists', () => {
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const source = generateCodexPlanReview({ skillName: 'plan-ceo-review', host: host.name, paths: HOST_PATHS[host.name] } as TemplateContext);
|
||||
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
|
||||
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual(['under_current_harness']);
|
||||
expect(source).toMatch(/SOURCE=in-host, OUTSIDE_STATUS=unavailable, and STATUS=clean or issues_found/);
|
||||
expect(source).not.toContain('Sections 1-10/11 and current report');
|
||||
}
|
||||
@@ -646,7 +645,7 @@ describe('outside-voice commitment queue', () => {
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const eng = generateCodexPlanReview({ host: host.name, paths: HOST_PATHS[host.name]!, skillName: 'plan-eng-review' } as TemplateContext);
|
||||
const provider = host.name === 'codex' ? 'Claude Code' : 'Codex';
|
||||
const mismatch = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const mismatch = 'under_current_harness';
|
||||
// B1: the heading also admits `unverified`.
|
||||
expect(eng).toContain('**If `CODEX_MODE: ready`');
|
||||
expect(eng).toContain(`— run ${provider}:**`);
|
||||
@@ -665,7 +664,7 @@ describe('outside-voice commitment queue', () => {
|
||||
expect(eng).not.toContain('codex exec');
|
||||
} else {
|
||||
expect(eng).toContain('codex exec');
|
||||
expect(eng).toMatch(/then follow \*\*Native fallback\*\*/i);
|
||||
expect(eng).toContain('_gstack_codex_role_ready exec');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@ import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
|
||||
callerExcerpt, callerProbeReadErrors, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
|
||||
QA_CALLER_CASES, QA_CALLER_TEST_MS,
|
||||
qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, unloadedHelperCommand, validateCallerEvidence,
|
||||
type CallerProbe, type CallerReceipt, type QaCallerFixture,
|
||||
@@ -87,6 +87,27 @@ function rebuildCheckpoints(observed: ReturnType<typeof evidence>) {
|
||||
}
|
||||
|
||||
describe('caller native-event observer controls', () => {
|
||||
test('captured instruction Read and probe in one provider response is not consumed instruction evidence', () => {
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-caller-read-probe-mtb1007a.json'), 'utf8'));
|
||||
const tools = callerTools(captured.events);
|
||||
const probe = tools.find(tool => tool.name === 'Bash')!;
|
||||
const resources = ['exploratory', 'system-functional'].map(id => [id, tools.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const);
|
||||
expect(resources[0][1]!.resultIndex).toBeLessThan(probe.index);
|
||||
expect(resources[0][1]!.messageId).toBe(probe.messageId);
|
||||
expect(callerProbeReadErrors(probe, resources)).toEqual([
|
||||
'probe preceded resource read: exploratory', 'probe preceded resource read: system-functional',
|
||||
]);
|
||||
expect(callerProbeReadErrors({ ...probe, messageId: 'separate-response-control' }, resources))
|
||||
.toEqual(['probe preceded resource read: system-functional']);
|
||||
const ordered = structuredClone(captured.events);
|
||||
const command = ordered.splice(2, 2);
|
||||
command[0].message.id = 'separate-response-control';
|
||||
ordered.push(...command);
|
||||
const ready = callerTools(ordered);
|
||||
expect(callerProbeReadErrors(ready.find(tool => tool.name === 'Bash')!,
|
||||
['exploratory', 'system-functional'].map(id => [id, ready.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const))).toEqual([]);
|
||||
});
|
||||
|
||||
test.each(['pending-result', 'same-event', 'same-message-id'])('method Read %s cannot authorize a dependent probe', ordering => {
|
||||
const observed = evidence();
|
||||
const events = observed.result.transcript as any[];
|
||||
|
||||
@@ -59,6 +59,14 @@ function assertPlanExecution(text: string, shared = generateQAExploratory({ host
|
||||
}
|
||||
|
||||
describe('QA probe entry and checkpoint gates', () => {
|
||||
test('the caller exposes the response boundary before the first QA instruction read', () => {
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
const text = generateQAReviewPreflight({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude });
|
||||
const beforeRead = text.slice(0, text.indexOf('{{QA_RESOURCE:exploratory}}'));
|
||||
expectMentions(beforeRead, [['reads', 'earlier', 'responses'], ['capture', 'probe'], ['never', 'batch', 'read']], 'caller entry gate');
|
||||
}
|
||||
});
|
||||
|
||||
test('the native CI preparation excerpt lacks the required acknowledged-read gate', () => {
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-functional-ci-36505065023.json'), 'utf8'));
|
||||
const output = captured.omittedReadEvents.flatMap(event => event.message.content)
|
||||
@@ -88,7 +96,9 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
|
||||
const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' ');
|
||||
const body = generateQAReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i);
|
||||
expect(preflight).toContain('`sections/...`');
|
||||
expect(preflight).toContain('`templates/...`');
|
||||
expectMentions(preflight, [['paths', 'installed qa skill.md', 'not', 'caller', 'product']], 'QA asset resolution');
|
||||
expect(body).toContain('Read QA\'s `sections/browser-setup.md`');
|
||||
expect(body).toContain('Read QA\'s `templates/functional-report-template.md`');
|
||||
if (skillName === 'review') {
|
||||
|
||||
@@ -366,7 +366,14 @@ describe('setup --team / --no-team / -q', () => {
|
||||
fs.writeFileSync(file, content, { mode: 0o755 });
|
||||
};
|
||||
try {
|
||||
for (const rel of ['setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names', 'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh', 'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts', 'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts']) {
|
||||
for (const rel of [
|
||||
'setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names',
|
||||
'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh',
|
||||
'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts',
|
||||
'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts',
|
||||
'lib/claude-code-migration.ts', 'lib/model-policy.ts', 'lib/model-catalog.ts',
|
||||
'lib/model-policy-notice.ts', 'lib/state-root.ts',
|
||||
]) {
|
||||
const dest = path.join(cwd, rel);
|
||||
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, rel), dest);
|
||||
|
||||
Reference in new issue
Block a user