From 92cfd07a79ed0f27fbcc57f2d61d00ec700eadb4 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 8 Oct 2026 01:00:08 -0700 Subject: [PATCH] v1.91.45.0 feat: configurable model tiers and advisory freshness (#3073) * feat: add configurable model tiers and advisory freshness * fix: disclose role selections before Claude review execution * fix: reject malformed model policies and keep model pins portable * fix: separate readiness diagnostics and bound version checks * fix: preserve actionable model failures and validate review inputs before probes * fix: retry released probe locks and retain late-rejection repair context * feat: finalize configurable model tiers and verified release repairs * fix: preserve strict model policy reads on Windows * fix: keep fixture probes read-only and clarify pending CEO decisions * fix: verify native terminal closure before bounded descriptor settlement --- .github/workflows/model-policy-freshness.yml | 49 ++ CHANGELOG.md | 12 + CONTRIBUTING.md | 5 +- README.md | 16 +- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- autoplan/SKILL.md | 23 +- autoplan/SKILL.md.tmpl | 2 + autoplan/sections/ceo-phase.md | 3 +- autoplan/sections/design-phase.md | 3 +- autoplan/sections/dx-phase.md | 3 +- autoplan/sections/eng-phase.md | 3 +- bin/gstack-codex-probe | 352 ++++++++- bin/gstack-config | 44 +- bin/gstack-models | 7 + claude-code/SKILL.md.tmpl | 3 + codex/SKILL.md | 23 +- codex/SKILL.md.tmpl | 23 +- codex/sections/challenge-mode.md | 8 +- codex/sections/challenge-mode.md.tmpl | 6 +- codex/sections/consult-mode.md | 4 +- codex/sections/consult-mode.md.tmpl | 2 +- codex/sections/review-mode.md | 8 +- codex/sections/review-mode.md.tmpl | 4 +- docs/model-policy.md | 241 ++++++ docs/skills.md | 14 +- docs/troubleshooting.md | 107 +++ lib/claude-code-migration.ts | 13 +- lib/claude-code.ts | 79 +- lib/model-catalog.ts | 43 ++ lib/model-policy-cli.ts | 256 +++++++ lib/model-policy-freshness-github.ts | 53 ++ lib/model-policy-freshness-parser-worker.ts | 6 + lib/model-policy-freshness-publication.ts | 127 +++ lib/model-policy-freshness-sources.ts | 232 ++++++ lib/model-policy-freshness.ts | 192 +++++ lib/model-policy-notice.ts | 44 ++ lib/model-policy.ts | 677 ++++++++++++++++ package.json | 2 +- plan-ceo-review/sections/review-sections.md | 68 +- plan-design-review/SKILL.md | 5 +- plan-devex-review/sections/review-sections.md | 70 +- plan-eng-review/sections/review-sections.md | 70 +- .../sections/review-sections.md.tmpl | 2 + review/SKILL.md | 6 +- scripts/harness-version.json | 80 +- scripts/model-policy-freshness.ts | 58 ++ scripts/preflight-codex-overlap.ts | 6 +- scripts/resolve-codex-generation-model.ts | 53 +- scripts/resolvers/constants.ts | 37 +- scripts/resolvers/design.ts | 4 +- scripts/resolvers/index.ts | 8 +- scripts/resolvers/outside-voice-steps.ts | 6 +- scripts/resolvers/outside-voice.ts | 27 +- scripts/resolvers/plan-review.ts | 7 + scripts/resolvers/qa.ts | 6 +- scripts/test-pr-profile.ts | 1 + ship/sections/review-army.md | 6 +- spec/SKILL.md | 1 + spec/SKILL.md.tmpl | 1 + spec/sections/gate-and-file.md | 5 +- spec/sections/gate-and-file.md.tmpl | 2 +- test/claude-code-migration.test.ts | 43 +- test/claude-code-skill.test.ts | 23 +- test/codex-hardening.test.ts | 31 +- test/codex-probe-cache.test.ts | 297 +++++++ test/codex-under-codex-detection.test.ts | 2 +- test/docsync-fault-interface.test.ts | 45 +- test/fixtures/model-policy-freshness.ts | 42 + .../anthropic-lifecycle.md | 16 + .../anthropic-models.md | 17 + .../openai-lifecycle.md | 24 + .../model-policy-freshness/openai-models.md | 13 + ...n-ceo-floor-unasked-decisions-872e154.json | 100 +++ .../qa-caller-read-probe-mtb1007a.json | 115 +++ test/helpers/docsync-fault-actor.ts | 2 +- test/helpers/free-fixtures-data.ts | 4 + test/helpers/pty/launch.ts | 1 + test/helpers/pty/runners/floor.ts | 2 +- test/helpers/qa-callers-fixture.ts | 10 +- test/model-catalog-isolation.test.ts | 82 ++ test/model-policy-cli.test.ts | 323 ++++++++ test/model-policy-freshness.test.ts | 245 ++++++ test/model-policy-workflow.test.ts | 547 +++++++++++++ test/model-policy.test.ts | 581 ++++++++++++++ test/model-role-invocation.test.ts | 723 ++++++++++++++++++ test/outside-voice-invocation.test.ts | 49 +- test/outside-voice-preflight.test.ts | 2 +- ...lan-ceo-floor-interactive-boundary.test.ts | 233 ++++++ test/plan-review-cases.test.ts | 7 +- test/pty-output-wake.test.ts | 84 ++ test/qa-exploratory-callers.test.ts | 23 +- test/qa-probe-gates.test.ts | 12 +- test/run-in-background-guidance.test.ts | 19 +- test/setup-claude-code-migration.test.ts | 2 +- test/team-mode.test.ts | 9 +- 96 files changed, 6539 insertions(+), 411 deletions(-) create mode 100644 .github/workflows/model-policy-freshness.yml create mode 100755 bin/gstack-models create mode 100644 docs/model-policy.md create mode 100644 lib/model-catalog.ts create mode 100644 lib/model-policy-cli.ts create mode 100644 lib/model-policy-freshness-github.ts create mode 100644 lib/model-policy-freshness-parser-worker.ts create mode 100644 lib/model-policy-freshness-publication.ts create mode 100644 lib/model-policy-freshness-sources.ts create mode 100644 lib/model-policy-freshness.ts create mode 100644 lib/model-policy-notice.ts create mode 100644 lib/model-policy.ts create mode 100644 scripts/model-policy-freshness.ts create mode 100644 scripts/resolvers/plan-review.ts create mode 100644 test/codex-probe-cache.test.ts create mode 100644 test/fixtures/model-policy-freshness.ts create mode 100644 test/fixtures/model-policy-freshness/anthropic-lifecycle.md create mode 100644 test/fixtures/model-policy-freshness/anthropic-models.md create mode 100644 test/fixtures/model-policy-freshness/openai-lifecycle.md create mode 100644 test/fixtures/model-policy-freshness/openai-models.md create mode 100644 test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json create mode 100644 test/fixtures/qa-caller-read-probe-mtb1007a.json create mode 100644 test/model-catalog-isolation.test.ts create mode 100644 test/model-policy-cli.test.ts create mode 100644 test/model-policy-freshness.test.ts create mode 100644 test/model-policy-workflow.test.ts create mode 100644 test/model-policy.test.ts create mode 100644 test/model-role-invocation.test.ts create mode 100644 test/plan-ceo-floor-interactive-boundary.test.ts diff --git a/.github/workflows/model-policy-freshness.yml b/.github/workflows/model-policy-freshness.yml new file mode 100644 index 000000000..b55f4bf2b --- /dev/null +++ b/.github/workflows/model-policy-freshness.yml @@ -0,0 +1,49 @@ +name: Model-policy freshness + +on: + schedule: + - cron: '0 11 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: model-policy-freshness + cancel-in-progress: false + +jobs: + advisory: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: 1.4.2 + - name: Install dependencies + run: bun install --frozen-lockfile + - name: Check and publish main advisory + if: github.ref == 'refs/heads/main' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json + run: bun scripts/model-policy-freshness.ts --publish + - name: Check non-main without publication + if: github.ref != 'refs/heads/main' + env: + MODEL_FRESHNESS_REPORT: ${{ runner.temp }}/model-policy-freshness.json + run: bun scripts/model-policy-freshness.ts + - name: Preserve evidence report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: model-policy-freshness-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/model-policy-freshness.json + if-no-files-found: error + retention-days: 90 diff --git a/CHANGELOG.md b/CHANGELOG.md index f48f68aef..9fa10af98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.91.45.0] - 2026-10-08 + +### Added +- Choose separate model tiers for planning and implementation. Independent plan reviews default to Fable 5.1 or GPT-6 Astra; implementation handoffs recommend Opus 5.5 or GPT-6.1 Sol without switching your session. Explicit model choices still win, and `plan_review_tier smart` or `host` provides an alternative to the frontier default. +- Inspect effective models, their sources and copyable pin/reset commands with `gstack-models`. Six settings use the existing configuration store, with fail-closed validation and a notice before the first affected review. See the [model-policy guide](docs/model-policy.md). +- Policy inspection reports native Windows paths and rejects non-directory or unreadable state roots rather than silently selecting defaults. +- Get weekly advisory checks of official model recommendations and retirement notices. One tracking issue retains source evidence and lifecycle history; upgrades remain human-reviewed, with no automatic model changes or paid benchmarking. + +### Fixed +- Codex model probes retain up to sixteen selections, serialize concurrent misses and preserve newer entries when another probe finishes later. Plan-review readiness and execution share one deadline, and failed model choices point to the setting that actually selected them. +- `/review` and `/ship` explicitly require prerequisite QA instructions to be read in earlier responses before a probe, including its evidence capture. + ## [1.91.42.0] - 2026-10-08 **/cso lifts its file-count ceiling: repositories with tens of thousands of files reach the audit.** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e12d09282..d99206723 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,8 +173,9 @@ defaults are `claude-fable-5-1`, resolved through `lib/eval-model.ts`: Warmup stays on `claude-haiku-4-5`; distill stays on `claude-haiku-4-5-20251001`. Explicit test and historical benchmark model -selections still win. Known frontier defaults are maintained in releases; -there is no automatic model discovery. Paid-run costs shown below are +selections still win. These eval defaults stay pinned independently from the +[runtime model tiers](docs/model-policy.md); the tier freshness workflow does +not discover or change evaluation models automatically. Paid-run costs shown below are historical estimates from before this default change, not measurements of the new defaults. diff --git a/README.md b/README.md index 11c7dee10..e2c99f362 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,18 @@ override applies to that run only; set `model` in your Codex `config.toml` to make it stick across upgrades. After changing your Codex model, rerun `./setup --host codex` to regenerate the skills. -**Which Codex model gstack uses.** For every Codex call (outside voices, -`/codex`, review and ship adversarial passes), gstack picks the model in this +**Plan-review and implementation tiers.** Plan workflows use an independent +`frontier` reviewer: Fable 5.1 from Codex, or GPT-6 Astra from Claude Code. +Implementation handoffs recommend the `smart` tier: Opus 5.5 or GPT-6.1 Sol. +Choose a different tier, pin either provider's model, or use `host` mode to +preserve native model settings. The read-only `gstack-models` command explains +the effective choices; changing these settings does not require regenerating +skills or switch your running session. A weekly maintenance check flags changed +official recommendations and retirement notices without automatically changing +models. See [model policy: setup, overrides and freshness](docs/model-policy.md). + +**Other Codex calls keep their existing selection.** Without the explicit +`plan-review` role (including ordinary review and ship adversarial passes), gstack picks the model in this order: a model you name for that request, then `GSTACK_CODEX_MODEL`, then `model` in your Codex `config.toml` (for native `codex review`, `review_model` first; a custom `CODEX_HOME` is honored), and only then gstack's default, @@ -177,7 +187,7 @@ repair message and reports the outside review as unavailable. It never silently switches to its default. Nested Codex reviews also run with installed skills hidden (`-c skills.include_instructions=false`), so a review cannot turn into a whole nested skill run. Runtime model selection is separate from the setup-time -behavioral profile above. `/claude-code` (`gstack-claude-code` +behavioral profile above. Without the plan-review role, `/claude-code` (`gstack-claude-code` on Codex) preserves Claude's configured model. Set `GSTACK_CLAUDE_MODEL=` or name a model in your request to override it for the invocation, including resumed consultations. See [eval defaults and overrides](CONTRIBUTING.md#testing--evals) diff --git a/VERSION b/VERSION index 4db65be2e..7705acdc4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.42.0 +1.91.45.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 46c28c53b..ec19a914c 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.42.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.45.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/autoplan/SKILL.md b/autoplan/SKILL.md index d2601cac3..d310539b8 100644 --- a/autoplan/SKILL.md +++ b/autoplan/SKILL.md @@ -771,26 +771,10 @@ elif [ -n "$_gstack_helper_error" ]; then elif ! "$_CODEX_PROBE" check-auth >/dev/null 2>&1; then _CODEX_MODE="not_authed"; "$_CODEX_PROBE" log-event codex_auth_failed 2>/dev/null || true else - # Free sandbox check before the paid probe; probe exit 2 = the CLI cannot run. - _CODEX_MP=0; _CODEX_PS="" + _CODEX_MP=0 "$_CODEX_PROBE" check-sandbox || _CODEX_MP=3 - for _CODEX_KIND in exec; do - [ "$_CODEX_MP" -eq 0 ] || break - _CODEX_PO=$("$_CODEX_PROBE" probe-model $_CODEX_KIND); _CODEX_MP=$?; printf '%s\n' "$_CODEX_PO" - case "$_CODEX_PO" in *"STATE: inconclusive"*) _CODEX_PS=inconclusive ;; *"STATE: rate_limited"*) _CODEX_PS=rate_limited ;; esac - done if [ "$_CODEX_MP" -eq 3 ]; then _CODEX_MODE="sandbox_unavailable" - elif [ "$_CODEX_MP" -eq 2 ]; then - _CODEX_MODE="broken_install" - elif [ "$_CODEX_MP" -eq 4 ]; then - _CODEX_MODE="quota_exhausted" - elif [ "$_CODEX_MP" -ne 0 ]; then - _CODEX_MODE="model_unusable" - elif [ "$_CODEX_PS" = inconclusive ]; then - _CODEX_MODE="unverified" - elif [ "$_CODEX_PS" = rate_limited ]; then - _CODEX_MODE="unverified (rate_limited)" else _CODEX_MODE="ready"; "$_CODEX_PROBE" check-version || true fi @@ -805,10 +789,11 @@ Branch on the echoed `CODEX_MODE`: - **`under_codex`** — stale artifact selected its own harness. Print: "Codex outside review unavailable: harness mismatch; no outside process started. Missing coverage. Repair: setup --host codex." Skip the outside invocation and follow the workflow's native-review instructions below. Conflicting inherited harness markers are not grounds to guess another provider. - **`not_authed`** — installed but no credentials. Print: "Codex not authenticated; outside coverage unavailable. Run `codex login` or set `$CODEX_API_KEY`." Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`broken_install`** — the CLI is on PATH but cannot execute (spawn ENOENT, non-executable binary, missing vendor payload). Print: "Codex is installed but its binary cannot run — Codex passes skipped. Reinstall: `npm install -g @openai/codex`." Relay the probe's HINT lines. Keep the required Claude adversarial pass; do not dispatch a duplicate. -- **`model_unusable`** — the selected model (see `CODEX_MODEL:`) is invalid or unavailable to the account (HTTP 400 on every call). Relay the probe's HINT lines and the fix (`GSTACK_CODEX_MODEL=` or config.toml `model`); never substitute a model. Keep the required Claude adversarial pass; do not dispatch a duplicate. The ~10s round trip is cached for 1h. +- **`model_unusable`** — the role invocation rejected policy, auth or model selection. Relay its reason and source-specific Repair/HINT lines, not a lower-priority setting. `AUTH_FAILED` needs `codex login`; never substitute a model. Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`quota_exhausted`** — Codex usage limit: relay the probe's lines verbatim (reset time, retry); no more Codex calls this run. Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`sandbox_unavailable`** — Codex's sandbox cannot start here (containers without user namespaces); the probe printed the reason and fix. No paid call ran; outside coverage is unavailable. Keep the required Claude adversarial pass; do not dispatch a duplicate. - **`ready`** or **`unverified`** — run the Codex pass below. `unverified` means the model check timed out or, with `(rate_limited)`, hit a 429; say so, and let the pass's own verdict decide. +Plan-review readiness probes the [policy](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md) model. Relay its diagnostics; never infer quota from the review block's exit status. Disabled/unavailable retains applicable native passes. Recheck each outside dispatch. Record provider and completed/unavailable/disabled/skipped per phase; CEO covers @@ -995,4 +980,6 @@ Retain the historical review-log skill ID; add `"host":"claude","outside_provide Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status, native completion, findings, and partial coverage. Replace N with actual counts. +**Implementation model:** relay model/source from `"$HOME/.claude/skills/gstack/bin/gstack-models" resolve --role implementation --provider anthropic`. gstack cannot change this session. Recommend only; no spawn or config edits unless asked. On error, relay its repair, not a model. [Policy setup](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md). + Suggest next step: `/ship` when ready to create the PR. diff --git a/autoplan/SKILL.md.tmpl b/autoplan/SKILL.md.tmpl index 287a64b6a..cc1e9e90a 100644 --- a/autoplan/SKILL.md.tmpl +++ b/autoplan/SKILL.md.tmpl @@ -464,4 +464,6 @@ disabled or skipped. Never carry success across phases/runs; preserve modelUsage Present a phase coverage table (CEO, design, DX, eng): host, outside provider/status, native completion, findings, and partial coverage. Replace N with actual counts. +{{IMPLEMENTATION_MODEL_HANDOFF}} + Suggest next step: `/ship` when ready to create the PR. diff --git a/autoplan/sections/ceo-phase.md b/autoplan/sections/ceo-phase.md index 3b9278560..073bdf023 100644 --- a/autoplan/sections/ceo-phase.md +++ b/autoplan/sections/ceo-phase.md @@ -113,10 +113,9 @@ _OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt" cat -- '' >"$_OUTSIDE_INPUT" || exit 1 _CODEX_PROBE="$HOME/.claude/skills/gstack/bin/gstack-codex-probe" -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -"$_CODEX_PROBE" check-sandbox || exit 1 "$_CODEX_PROBE" show-first-use-notice _OUTSIDE_EXIT=0 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$? diff --git a/autoplan/sections/design-phase.md b/autoplan/sections/design-phase.md index ee0df9751..f8e043365 100644 --- a/autoplan/sections/design-phase.md +++ b/autoplan/sections/design-phase.md @@ -72,10 +72,9 @@ _OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt" cat -- '' >"$_OUTSIDE_INPUT" || exit 1 _CODEX_PROBE="$HOME/.claude/skills/gstack/bin/gstack-codex-probe" -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -"$_CODEX_PROBE" check-sandbox || exit 1 "$_CODEX_PROBE" show-first-use-notice _OUTSIDE_EXIT=0 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$? diff --git a/autoplan/sections/dx-phase.md b/autoplan/sections/dx-phase.md index b0af8a69c..73b0e2510 100644 --- a/autoplan/sections/dx-phase.md +++ b/autoplan/sections/dx-phase.md @@ -76,10 +76,9 @@ _OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt" cat -- '' >"$_OUTSIDE_INPUT" || exit 1 _CODEX_PROBE="$HOME/.claude/skills/gstack/bin/gstack-codex-probe" -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -"$_CODEX_PROBE" check-sandbox || exit 1 "$_CODEX_PROBE" show-first-use-notice _OUTSIDE_EXIT=0 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$? diff --git a/autoplan/sections/eng-phase.md b/autoplan/sections/eng-phase.md index 5511e0c26..6ba893c07 100644 --- a/autoplan/sections/eng-phase.md +++ b/autoplan/sections/eng-phase.md @@ -65,10 +65,9 @@ _OUTSIDE_INPUT="$_OUTSIDE_TMP/prompt" cat -- '' >"$_OUTSIDE_INPUT" || exit 1 _CODEX_PROBE="$HOME/.claude/skills/gstack/bin/gstack-codex-probe" -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -"$_CODEX_PROBE" check-sandbox || exit 1 "$_CODEX_PROBE" show-first-use-notice _OUTSIDE_EXIT=0 "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$_OUTSIDE_TMP/text" <"$_OUTSIDE_INPUT" >"$_OUTSIDE_TMP/events" 2>"$_OUTSIDE_TMP/stderr" || _OUTSIDE_EXIT=$? diff --git a/bin/gstack-codex-probe b/bin/gstack-codex-probe index c3f12c4e6..f09173b7b 100755 --- a/bin/gstack-codex-probe +++ b/bin/gstack-codex-probe @@ -10,6 +10,10 @@ # show-sandbox CODEX_SANDBOX: # check-sandbox exit 0, or 3 with the reason on stderr # probe-model [--model ] CODEX_PROBE_STATE:; exit 0, 1, 2 or 4 +# role-ready [--model ] [--cwd ] +# plan-review select, auth, sandbox and probe once: +# CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:, +# CODEX_PROBE_STATE:; exit 0, 1, 2, 3 or 4 # check-version warnings on stderr; exit 0 # show-first-use-notice notice on stderr; exit 0 # run-with-timeout the command's exit code; 124 on timeout @@ -17,6 +21,10 @@ # log-hang silent; exit 0 # help usage; an unknown subcommand exits 64 # +# One deadline: an exported _CODEX_DEADLINE (epoch seconds) caps every +# supervised command at the time left, and one that would start after it does +# not run (exit 124). role-ready spends at most half of that time. +# # Sourcing (compatibility window): skills rendered before the executed form # source this file and call the functions directly. That still works, prints # one deprecation line on stderr, and is removed in a release on or after @@ -30,6 +38,8 @@ # _gstack_codex_first_use_notice — once per machine: provider, account, how to turn off # _gstack_codex_model_probe — round-trip probe of the selected model (#2477); # exit 4 = the account's Codex usage limit +# _gstack_codex_lock_acquire — bounded mkdir lock with abandoned-owner recovery +# _gstack_codex_role_ready — plan-review selection, auth, sandbox and model probe (role-ready) # _gstack_codex_version_check — warn on known-bad Codex CLI versions # _gstack_codex_timeout_wrapper — gtimeout -> timeout -> bash-native watchdog # _gstack_codex_log_event — telemetry emission to ~/.gstack/analytics/ @@ -134,33 +144,74 @@ _gstack_codex_select_model() { # One selection record per invocation kind, shared by the probe and the # dispatch flag in the same block: explicit request ($2), GSTACK_CODEX_MODEL, # Codex config.toml (review_model first for native review), gstack default. + # With $3 = plan-review the plan-review policy ranks the choice instead and + # its record carries role, tier, source and source-aware repair; $4 names the + # working-directory context forwarded to the shared role resolver. # The TOML is read by the TypeScript resolver; bash never parses it. - local _kind="${1:-exec}" _explicit="${2:-}" _line _rc + local _kind="${1:-exec}" _explicit="${2:-}" _role="${3:-}" _cwd="${4:-}" _line _rc _rest case "$_kind" in exec|review) ;; *) echo "ERROR: _gstack_codex_select_model takes exec or review" >&2; return 2 ;; esac + case "$_role" in ''|plan-review) ;; *) echo "ERROR: _gstack_codex_select_model role must be plan-review" >&2; return 2 ;; esac if ! command -v bun >/dev/null 2>&1; then echo "CODEX_MODEL: unresolved — bun is not on PATH, so gstack cannot read the Codex model choice. Reinstall gstack with ./setup; no Codex call was made." >&2 return 1 fi - if [ -n "$_explicit" ]; then - _line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind" --explicit "$_explicit") - else - _line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" --runtime "$_kind") - fi + # Function-local argv: each value stays one literal argument in bash and zsh. + set -- --runtime "$_kind" + [ -n "$_role" ] && set -- "$@" --role "$_role" + [ -n "$_explicit" ] && set -- "$@" --explicit "$_explicit" + [ -n "$_cwd" ] && set -- "$@" --cwd "$_cwd" + _line=$(bun run "$_GSTACK_CODEX_BIN/../scripts/resolve-codex-generation-model.ts" "$@") _rc=$? _GSTACK_CODEX_SEL="${_line%%$'\t'*}" - _GSTACK_CODEX_SEL_SRC="${_line#*$'\t'}" + _rest="${_line#*$'\t'}" + _GSTACK_CODEX_SEL_TIER="" + _GSTACK_CODEX_SEL_REPAIR="" + if [ -n "$_role" ]; then + _GSTACK_CODEX_SEL_SRC="${_rest%%$'\t'*}" + _rest="${_rest#*$'\t'}" + _GSTACK_CODEX_SEL_TIER="${_rest%%$'\t'*}" + _GSTACK_CODEX_SEL_REPAIR="${_rest#*$'\t'}" + else + _GSTACK_CODEX_SEL_SRC="$_rest" + fi case "$_GSTACK_CODEX_SEL" in ''|*[!A-Za-z0-9._:/-]*) _rc=1 ;; esac if [ "$_rc" -ne 0 ] || [ "${#_GSTACK_CODEX_SEL}" -gt 100 ]; then _GSTACK_CODEX_SEL="" _GSTACK_CODEX_SEL_SRC="" + _GSTACK_CODEX_SEL_ROLE="" echo "CODEX_MODEL: invalid — Codex outside review unavailable; missing coverage. No Codex call was made. Fix the model choice above; gstack never substitutes its default for it." >&2 return 1 fi _GSTACK_CODEX_SEL_KIND="$_kind" - echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2 + _GSTACK_CODEX_SEL_ROLE="$_role" + _GSTACK_CODEX_SEL_CWD="$_cwd" + if [ -n "$_role" ]; then + echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; role: $_role, tier: $_GSTACK_CODEX_SEL_TIER; source: $_GSTACK_CODEX_SEL_SRC)" >&2 + else + echo "CODEX_MODEL: $_GSTACK_CODEX_SEL ($_kind; source: $_GSTACK_CODEX_SEL_SRC)" >&2 + fi _gstack_codex_sandbox_mode } +_gstack_codex_role_ready() { + # $1 exec|review, $2 target repository, $3 a model named for this request. + # Resolves the plan-review model once and checks auth, the sandbox and that + # model; the caller dispatches the same record. Under _CODEX_DEADLINE these + # checks get the first half of the time left, so the dispatch keeps the rest. + local _end="${_CODEX_DEADLINE:-}" _now + case "$_end" in + ''|*[!0-9]*) local _CODEX_DEADLINE="" ;; + *) _now=$(date +%s 2>/dev/null || echo 0); local _CODEX_DEADLINE=$(( _now + (_end - _now) / 2 )) ;; + esac + _gstack_codex_select_model "$1" "${3:-}" plan-review "${2:-}" || return 1 + _gstack_codex_auth_probe || return 1 + _gstack_codex_sandbox_preflight || return $? + _gstack_codex_version_check + _gstack_codex_model_probe "$1" || return $? + [ -z "${_GSTACK_CODEX_SEL_REPAIR:-}" ] || echo "HINT: If the review rejects the selected model, repair its winning source: $_GSTACK_CODEX_SEL_REPAIR." + return 0 +} + # --- Sandbox (B1) ----------------------------------------------------------- _gstack_codex_sandbox_mode() { @@ -266,7 +317,7 @@ _gstack_codex_model_probe() { # auth just measures the auth failure again. local _kind="${1:-exec}" if [ -z "${_GSTACK_CODEX_SEL:-}" ] || [ "${_GSTACK_CODEX_SEL_KIND:-}" != "$_kind" ]; then - if ! _gstack_codex_select_model "$_kind"; then + if ! _gstack_codex_select_model "$_kind" "" "${_GSTACK_CODEX_SEL_ROLE:-}" "${_GSTACK_CODEX_SEL_CWD:-}"; then echo "MODEL_UNUSABLE" echo "HINT: the requested Codex model is not a valid model id; fix it as described above." return 1 @@ -306,36 +357,49 @@ _gstack_codex_model_probe() { local _now _now=$(date +%s 2>/dev/null || echo 0) local _docs="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md" - if [ -n "$_cache" ] && [ -f "$_cache" ]; then - local _c_line _c_status _c_ts _c_sig - _c_line=$(head -1 "$_cache" 2>/dev/null) - _c_status=$(printf '%s' "$_c_line" | cut -d' ' -f1) - _c_ts=$(printf '%s' "$_c_line" | cut -d' ' -f2) - _c_sig=$(printf '%s' "$_c_line" | cut -d' ' -f3) - case "$_c_ts" in ''|*[!0-9]*) _c_ts=0 ;; esac - if [ "$_c_status" = "MODEL_OK" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 3600 ]; then - echo "MODEL_OK (cached)" - return 0 + # Up to 16 signatures share one cache file. Same-signature misses serialize + # on a bounded lock and recheck the cache, so concurrent callers pay for one + # round trip. A lock timeout or unreadable cache is unverified readiness, + # never a duplicate paid probe; the review's own result still decides. + local _lock="" _rc _wait="${_GSTACK_CODEX_LOCK_WAIT:-45}" + # Under a deadline, wait only while the 30-second round trip still fits. + case "${_CODEX_DEADLINE:-}" in + ''|*[!0-9]*) ;; + *) [ $((_CODEX_DEADLINE - _now - 30)) -lt "$_wait" ] && _wait=$((_CODEX_DEADLINE - _now - 30)) + [ "$_wait" -ge 0 ] || _wait=0 ;; + esac + if [ -n "$_cache" ]; then + _gstack_codex_probe_cache_hit; _rc=$? + [ "$_rc" -lt 10 ] && return "$_rc" + if [ "$_rc" -eq 11 ]; then _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; fi + _lock="$_gstack_home/.codex-model-probe.locks/$_sig" + if ! mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null; then + _gstack_codex_probe_unverified "cannot create the probe lock under $_gstack_home"; return 0 fi - [ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && _c_sig="retry" - if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then - echo "MODEL_QUOTA_EXHAUSTED (cached)" - sed -n '2,4p' "$_cache" 2>/dev/null - echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted" - return 4 - fi - if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ "$_c_sig" = "$_sig" ] && [ $((_now - _c_ts)) -lt 900 ]; then - echo "MODEL_UNUSABLE (cached)" - echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." - echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." - return 1 + _gstack_codex_lock_acquire "$_lock" "$_wait" "${_GSTACK_CODEX_LOCK_STALE:-90}"; _rc=$? + if [ "$_rc" -eq 1 ]; then _gstack_codex_probe_unverified "another probe of this model still holds its lock"; return 0; fi + if [ "$_rc" -ne 0 ]; then _gstack_codex_probe_unverified "cannot record ownership of the probe lock under $_gstack_home"; return 0; fi + _now=$(date +%s 2>/dev/null || echo "$_now") + _gstack_codex_probe_cache_hit; _rc=$? + if [ "$_rc" -lt 10 ] || [ "$_rc" -eq 11 ]; then + _gstack_codex_lock_release "$_lock" + [ "$_rc" -eq 11 ] && { _gstack_codex_probe_unverified "probe cache $_cache is unreadable"; return 0; } + return "$_rc" fi fi + _gstack_codex_probe_round_trip; _rc=$? + [ -n "$_lock" ] && _gstack_codex_lock_release "$_lock" + return "$_rc" +} + +_gstack_codex_probe_round_trip() { + # The paid round trip and its classification. Called only by + # _gstack_codex_model_probe; reads its locals (_model, _sig, _cache, ...). local _out _code _out=$(_gstack_codex_timeout_wrapper 30 codex exec --skip-git-repo-check -s read-only -c "model=\"$_model\"" -c 'skills.include_instructions=false' "reply OK" &1) _code=$? if [ "$_code" -eq 0 ]; then - [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_OK %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; } + _gstack_codex_probe_cache_store MODEL_OK echo "MODEL_OK" return 0 fi @@ -355,7 +419,7 @@ _gstack_codex_model_probe() { fi _q_lines=$(printf '%s' "$_err" | grep -iE "usage limit|insufficient_quota|exceeded your current quota|quota exceeded" | head -3) if [ -n "$_q_lines" ]; then - [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_QUOTA_EXHAUSTED %s %s\n%s\n' "$_now" "$_sig" "$_q_lines" > "$_cache" 2>/dev/null; } + _gstack_codex_probe_cache_store MODEL_QUOTA_EXHAUSTED "$_q_lines" echo "MODEL_QUOTA_EXHAUSTED" printf '%s\n' "$_q_lines" echo "HINT: Codex refused the call for this account's usage limit (Codex's line above names the reset time); the model choice is fine. Outside coverage is unavailable: gstack skips Codex for 15 minutes. Retry now: GSTACK_CODEX_PROBE_RETRY=1${_cache:+, or delete $_cache}. $_docs#codex-quota-exhausted" @@ -377,7 +441,7 @@ _gstack_codex_model_probe() { # base_url answers a bare 404; neither is network luck, so neither may fall # through to the inconclusive branch below and report ready (#2843). if printf '%s' "$_out" | grep -qiE 'model.{0,40}is not supported|"status":[[:space:]]*40[04]|requires a newer version of codex|does not exist or you do not have access|model.{0,40}does not exist|404 not found|status:?[[:space:]]*404'; then - [ -n "$_cache" ] && { mkdir -p "$_gstack_home" 2>/dev/null; printf 'MODEL_UNUSABLE %s %s\n' "$_now" "$_sig" > "$_cache" 2>/dev/null; } + _gstack_codex_probe_cache_store MODEL_UNUSABLE echo "MODEL_UNUSABLE" printf '%s\n' "$_out" | grep -iE "model|404" | head -3 echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." @@ -386,10 +450,14 @@ _gstack_codex_model_probe() { _gstack_codex_log_event "codex_cli_outdated" 2>/dev/null || true elif printf '%s' "$_out" | grep -qiE 'does not exist|404'; then echo "HINT: Codex answered 404: either the model is retired or not visible to this account, or a custom provider's base_url in $_codex_home/config.toml is wrong." - echo "HINT: choose a current model (GSTACK_CODEX_MODEL= or model in config.toml), or fix that provider's base_url." + if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then + echo "HINT: choose a current model ($_GSTACK_CODEX_SEL_REPAIR), or fix that provider's base_url." + else + echo "HINT: choose a current model (GSTACK_CODEX_MODEL= or model in config.toml), or fix that provider's base_url." + fi _gstack_codex_log_event "codex_model_retired" 2>/dev/null || true else - echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in $_codex_home/config.toml, or name one for this request." + _gstack_codex_model_repair_hint _gstack_codex_log_event "codex_model_unusable" 2>/dev/null || true fi return 1 @@ -420,6 +488,170 @@ _gstack_codex_model_probe() { return 0 } +_gstack_codex_model_repair_hint() { + # Repair for the record actually selected: a plan-review record names its + # winning source (request, env, tier override, catalog or host mode). + if [ -n "${_GSTACK_CODEX_SEL_REPAIR:-}" ]; then + echo "HINT: choose a model your account can use: $_GSTACK_CODEX_SEL_REPAIR" + else + echo "HINT: choose a model your account can use: set GSTACK_CODEX_MODEL=, set model in ${_codex_home:-\$CODEX_HOME}/config.toml, or name one for this request." + fi +} + +_gstack_codex_probe_unverified() { + _GSTACK_CODEX_PROBE_STATE="inconclusive" + echo "MODEL_PROBE_INCONCLUSIVE ($1) — CODEX_MODE: unverified; no duplicate probe was made, and the review's own result is still checked." +} + +_gstack_codex_probe_cache_hit() { + # Cache file: one "STATUS EPOCH SIGNATURE" line per entry, newest last, each + # followed by its tab-prefixed Codex lines. The pre-v2 one-entry file has the + # same first line, so it migrates by TTL; its unprefixed lines are ignored. + # Returns 0/1/4 after printing a hit, 10 on a miss, 11 when unreadable. + [ -e "$_cache" ] || return 10 + { [ -f "$_cache" ] && [ -r "$_cache" ]; } || return 11 + local _hit _head _c_status _c_ts _detail="" + _hit=$(awk -v sig="$_sig" ' + /^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ { cur = ($3 == sig); if (cur) out = $1 " " $2; next } + /^\t/ { if (cur) out = out "\n" substr($0, 2); next } + { cur = 0 } + END { if (out != "") print out }' "$_cache" 2>/dev/null) || return 11 + [ -n "$_hit" ] || return 10 + _head="${_hit%%$'\n'*}" + [ "$_head" != "$_hit" ] && _detail="${_hit#*$'\n'}" + _c_status="${_head%% *}" + _c_ts="${_head#* }" + case "$_c_ts" in ''|*[!0-9]*) return 10 ;; esac + [ "$_c_ts" -le "$_now" ] || return 10 + if [ "$_c_status" = "MODEL_OK" ] && [ $((_now - _c_ts)) -lt 3600 ]; then + echo "MODEL_OK (cached)" + return 0 + fi + [ "${GSTACK_CODEX_PROBE_RETRY:-}" = "1" ] && return 10 + if [ "$_c_status" = "MODEL_QUOTA_EXHAUSTED" ] && [ $((_now - _c_ts)) -lt 900 ]; then + echo "MODEL_QUOTA_EXHAUSTED (cached)" + [ -n "$_detail" ] && printf '%s\n' "$_detail" + echo "HINT: Codex refused this account's calls for its usage limit (Codex's line above names the reset time); the model choice is fine. No Codex call was made: gstack skips Codex for $(( (_c_ts + 900 - _now + 59) / 60 )) more minute(s). Retry now: GSTACK_CODEX_PROBE_RETRY=1, or delete $_cache. $_docs#codex-quota-exhausted" + return 4 + fi + if [ "$_c_status" = "MODEL_UNUSABLE" ] && [ $((_now - _c_ts)) -lt 900 ]; then + echo "MODEL_UNUSABLE (cached)" + echo "HINT: gstack requested model '$_model' (source: $_GSTACK_CODEX_SEL_SRC)." + _gstack_codex_model_repair_hint + return 1 + fi + return 10 +} + +_gstack_codex_probe_cache_store() { + # $1 status, $2 optional Codex lines. Rewrites the file under a short write + # lock and publishes it with one rename: this signature's entry replaces any + # older one, expired entries drop, and at most 16 entries remain. A failure + # leaves the previous file intact; the probe's own result still stands. + # Timestamps are publication time, read under the write lock: a slow round + # trip must not judge entries other probes published meanwhile as future. + [ -n "$_cache" ] || return 0 + [ -d "$_cache" ] && return 1 + local _w="$_gstack_home/.codex-model-probe.locks/write" _tmp="$_cache.tmp.$$" _wrc=0 _pub + mkdir -p "$_gstack_home/.codex-model-probe.locks" 2>/dev/null || return 1 + _gstack_codex_lock_acquire "$_w" 5 30 || return 1 + _pub=$(date +%s 2>/dev/null || echo "$_now") + : > "$_tmp" 2>/dev/null || _wrc=1 + if [ "$_wrc" -eq 0 ] && [ -f "$_cache" ]; then + awk -v now="$_pub" -v sig="$_sig" -v keep=15 ' + /^(MODEL_OK|MODEL_UNUSABLE|MODEL_QUOTA_EXHAUSTED) [0-9]+ [0-9]+$/ { + ttl = ($1 == "MODEL_OK") ? 3600 : 900 + cur = ($3 != sig && $2 <= now && now - $2 < ttl) + if (cur) entry[++n] = $0 + next + } + /^\t/ { if (cur) entry[n] = entry[n] "\n" $0; next } + { cur = 0 } + END { for (i = (n > keep ? n - keep + 1 : 1); i <= n; i++) print entry[i] }' "$_cache" > "$_tmp" 2>/dev/null || _wrc=1 + fi + if [ "$_wrc" -eq 0 ]; then + { + printf '%s %s %s\n' "$1" "$_pub" "$_sig" + [ -z "${2:-}" ] || printf '%s\n' "$2" | awk '{ print "\t" $0 }' + } >> "$_tmp" 2>/dev/null && mv -f "$_tmp" "$_cache" 2>/dev/null || _wrc=1 + fi + rm -f "$_tmp" 2>/dev/null + _gstack_codex_lock_release "$_w" + return "$_wrc" +} + +# --- Bounded lock ----------------------------------------------------------- + +_gstack_codex_lock_acquire() { + # $1 lock directory, $2 seconds to wait, $3 seconds after which even a live + # owner is abandoned. mkdir is atomic on every platform bash runs on. A dead + # owner on this host is recovered at once; a missing owner file is the + # creation window and becomes stale after 5 seconds. 0 = held, 1 = timed + # out, 2 = the lock or its owner record cannot be created. + local _dir="$1" _tries=$(( ${2:-45} * 10 )) _stale="${3:-90}" _host _owner _pid _ohost _ts _now _i=0 _dead + _host=$(uname -n 2>/dev/null || echo unknown) + while :; do + if mkdir "$_dir" 2>/dev/null; then + printf '%s %s %s\n' "$$" "$_host" "$(date +%s 2>/dev/null || echo 0)" > "$_dir/owner" 2>/dev/null && return 0 + rm -f "$_dir/owner" 2>/dev/null + rmdir "$_dir" 2>/dev/null + return 2 + fi + if [ ! -d "$_dir" ]; then + { [ -e "$_dir" ] || [ -L "$_dir" ]; } && return 2 + { [ -d "${_dir%/*}" ] && [ -w "${_dir%/*}" ] && [ -x "${_dir%/*}" ]; } || return 2 + [ "$_i" -ge "$_tries" ] && return 2 + _i=$((_i + 1)) + sleep 0.1 + continue + fi + _now=$(date +%s 2>/dev/null || echo 0) + _owner=$(cat "$_dir/owner" 2>/dev/null) + _dead=0 + if [ -n "$_owner" ]; then + read -r _pid _ohost _ts <<< "$_owner" + case "$_pid" in ''|*[!0-9]*) _pid=0 ;; esac + case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac + if [ "$_ohost" = "$_host" ] && ! kill -0 "$_pid" 2>/dev/null; then _dead=1; fi + [ $((_now - _ts)) -ge "$_stale" ] && _dead=1 + else + _ts=$(stat -c %Y "$_dir" 2>/dev/null || stat -f %m "$_dir" 2>/dev/null || echo 0) + case "$_ts" in ''|*[!0-9]*) _ts=0 ;; esac + [ $((_now - _ts)) -ge 5 ] && _dead=1 + fi + [ "$_dead" -eq 1 ] && _gstack_codex_lock_break "$_dir" "$_owner" + [ "$_i" -ge "$_tries" ] && return 1 + _i=$((_i + 1)) + sleep 0.1 + done +} + +_gstack_codex_lock_break() { + # Breakers serialize on $1.break and remove the lock only if it still has the + # owner they judged abandoned, so a lock re-taken meanwhile survives. + local _dir="$1" _seen="$2" _bt _now + if mkdir "$_dir.break" 2>/dev/null; then + [ "$(cat "$_dir/owner" 2>/dev/null)" = "$_seen" ] && rm -rf "$_dir" 2>/dev/null + rmdir "$_dir.break" 2>/dev/null + return 0 + fi + _now=$(date +%s 2>/dev/null || echo 0) + _bt=$(stat -c %Y "$_dir.break" 2>/dev/null || stat -f %m "$_dir.break" 2>/dev/null || echo "$_now") + case "$_bt" in ''|*[!0-9]*) _bt="$_now" ;; esac + [ $((_now - _bt)) -ge 10 ] && rmdir "$_dir.break" 2>/dev/null + return 0 +} + +_gstack_codex_lock_release() { + # Remove only a lock this shell still owns. + local _host + _host=$(uname -n 2>/dev/null || echo unknown) + case "$(cat "$1/owner" 2>/dev/null)" in + "$$ $_host "*) rm -rf "$1" 2>/dev/null ;; + esac + return 0 +} + # --- Version check ---------------------------------------------------------- _gstack_codex_version_check() { @@ -431,9 +663,13 @@ _gstack_codex_version_check() { # Capture the code from codex, not from `head` — a pipeline reports the LAST # command's status, which is why a CLI that only ever printed a spawn error # still read as healthy here (#2742). Keep stderr: it carries the diagnosis. - _ver=$(codex --version 2>&1) + _ver=$(_gstack_codex_timeout_wrapper 5 codex --version &1) _vcode=$? _ver=$(printf '%s' "$_ver" | head -1) + if [ "$_vcode" -eq 124 ]; then + echo 'WARN: `codex --version` timed out; version unknown. The invocation keeps its existing deadline.' + return 0 + fi # Only a NON-ZERO exit is evidence of a broken CLI. Empty-but-successful # output stays silent by design (a CLI may legitimately print nothing), which # the "empty output → OK" case in this file's suite pins. @@ -461,10 +697,22 @@ _gstack_codex_timeout_wrapper() { # (_GSTACK_CODEX_KILL_AFTER seconds, default 10), so the deadline always # ends the provider before the caller's outer tool gate (#2776). Either # way a deadline reports timeout(1)'s exit 124; output already written stays. + # An exported _CODEX_DEADLINE (epoch seconds) caps the duration at the time + # left; past it the command never starts. local _duration="$1" shift local _grace="${_GSTACK_CODEX_KILL_AFTER:-10}" - local _to _rc _start + local _to _rc _start _left + case "${_CODEX_DEADLINE:-}" in + ''|*[!0-9]*) ;; + *) _left=$(( _CODEX_DEADLINE - $(date +%s 2>/dev/null || echo 0) )) + if [ "$_left" -le 0 ]; then + echo "gstack: the invocation's deadline passed before this command started; nothing ran." >&2 + return 124 + fi + _start="${_duration%%.*}" + [ "${_start:-0}" -ge "$_left" ] && _duration="$_left" ;; + esac _to=$(command -v gtimeout 2>/dev/null || command -v timeout 2>/dev/null || echo "") if [ -n "$_to" ]; then _start=$(date +%s 2>/dev/null || echo 0) @@ -563,6 +811,9 @@ _gstack_codex_usage() { ' show-sandbox print CODEX_SANDBOX: read-only (or danger-full-access)' \ ' check-sandbox free sandbox check; exit 0, or 3 with the reason on stderr' \ ' probe-model [--model ] paid round trip; print CODEX_PROBE_STATE:; exit 0, 1, 2 or 4' \ + ' role-ready [--model ] [--cwd ]' \ + ' plan-review selection, auth, sandbox and probe once; print' \ + ' CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:, CODEX_PROBE_STATE:' \ ' check-version warn on stderr about known-bad Codex CLI versions; exit 0' \ ' show-first-use-notice once per machine, say where reviews go; exit 0' \ ' run-with-timeout run cmd under a deadline; its exit code, 124 on timeout' \ @@ -584,11 +835,28 @@ _gstack_codex_kind_args() { fi } +# Parses " [--model ] [--cwd ]" for role-ready. +_gstack_codex_role_args() { + _GSTACK_CODEX_ARG_KIND="${1:-}" + _GSTACK_CODEX_ARG_MODEL="" + _GSTACK_CODEX_ARG_CWD="" + case "$_GSTACK_CODEX_ARG_KIND" in exec|review) shift ;; *) return 64 ;; esac + while [ "$#" -gt 0 ]; do + { [ "$#" -ge 2 ] && [ -n "$2" ]; } || return 64 + case "$1" in + --model) _GSTACK_CODEX_ARG_MODEL="$2" ;; + --cwd) _GSTACK_CODEX_ARG_CWD="$2" ;; + *) return 64 ;; + esac + shift 2 + done +} + _gstack_codex_main() { local _cmd="${1:-}" _rc [ "$#" -gt 0 ] && shift case "$_cmd" in - log-event|probe-model|check-sandbox|check-version) + log-event|probe-model|role-ready|check-sandbox|check-version) # Telemetry is the caller's choice when it exports _TEL, else gstack-config's. [ -n "${_TEL+set}" ] || _TEL=$("$_GSTACK_CODEX_BIN/gstack-config" get telemetry 2>/dev/null || echo off) ;; esac @@ -622,6 +890,12 @@ _gstack_codex_main() { *) echo "CODEX_PROBE_STATE: error" ;; esac return "$_rc" ;; + role-ready) + _gstack_codex_role_args "$@" || { _gstack_codex_usage >&2; return 64; } + _gstack_codex_role_ready "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_CWD" "$_GSTACK_CODEX_ARG_MODEL" >&2 + _rc=$? + [ "$_rc" -eq 0 ] || return "$_rc" + printf 'CODEX_SEL: %s\nCODEX_SEL_KIND: %s\nCODEX_SANDBOX: %s\nCODEX_PROBE_STATE: %s\n' "$_GSTACK_CODEX_SEL" "$_GSTACK_CODEX_SEL_KIND" "$_GSTACK_CODEX_SANDBOX" "${_GSTACK_CODEX_PROBE_STATE:-ok}" ;; check-version) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_version_check >&2; return 0 ;; show-first-use-notice) [ "$#" -eq 0 ] || { _gstack_codex_usage >&2; return 64; }; _gstack_codex_first_use_notice; return 0 ;; run-with-timeout) diff --git a/bin/gstack-config b/bin/gstack-config index e362e74ad..84c46c57f 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -151,6 +151,18 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # ID and re-renders; ./setup --claude-model claude # # returns to the generic overlay. # +# ─── Model policy (gstack-models shows the effective choices) ─────── +# Plan review tier: frontier | smart | host. host delegates to the CLI settings; +# GSTACK_CLAUDE_MODEL / GSTACK_CODEX_MODEL still win. +# plan_review_tier: frontier +# Implementation recommendation: frontier | smart. Never switches your session. +# implementation_tier: smart +# Optional pins: model_frontier_claude, model_frontier_openai, +# model_smart_claude and model_smart_openai. Leave pins absent for catalog defaults; +# empty pins are invalid. gstack-models list prints copyable set/unset commands. +# Invalid writes are rejected and preserve the stored value. +# Docs: docs/model-policy.md +# # ─── Advanced ──────────────────────────────────────────────────────── # codex_reviews: enabled # Workflow outside review (Codex or Claude Code by harness). # # enabled: /review, /ship, /document-release, plan reviews, @@ -283,6 +295,9 @@ lookup_default() { ship_measure_sweep_cases) echo "5" ;; ship_measure_sweep_budget_usd) echo "150" ;; ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in) + plan_review_tier) echo "frontier" ;; # frontier | smart | host — lib/model-policy.ts reads it strictly + implementation_tier) echo "smart" ;; # frontier | smart + model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai) echo "" ;; # empty = shipped catalog ID (gstack-models list) ship_greptile_early) echo "true" ;; # true | false — /ship's early Greptile PR # Unknown key: exit non-zero instead of printing "". The fallback pattern # the preambles use, @@ -684,6 +699,27 @@ case "${1:-}" in echo "Error: codex_reviews '$VALUE' not recognized. Valid values: enabled, disabled. Existing value left unchanged." >&2 exit 1 fi + case "$KEY" in + plan_review_tier|implementation_tier|model_frontier_claude|model_frontier_openai|model_smart_claude|model_smart_openai) + _mp_doc="https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#model-policy-config" + VALUE="${VALUE#"${VALUE%%[![:space:]]*}"}" + VALUE="${VALUE%"${VALUE##*[![:space:]]}"}" + _mp_valid=1 + case "$KEY" in + plan_review_tier) case "$VALUE" in frontier|smart|host) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart, host." ;; esac ;; + implementation_tier) case "$VALUE" in frontier|smart) ;; *) _mp_valid="" ; _mp_rule="Valid values: frontier, smart." ;; esac ;; + *_claude) { [ "${#VALUE}" -le 512 ] && printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[][A-Za-z0-9._:/@+-]+$'; } || { _mp_valid=""; _mp_rule="An Anthropic model ID is 1-512 characters from A-Z a-z 0-9 . _ : / @ + [ ] -."; } ;; + *_openai) printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[A-Za-z0-9._:/-]{1,100}$' || { _mp_valid=""; _mp_rule="An OpenAI model ID is 1-100 characters from A-Z a-z 0-9 . _ : / -."; } ;; + esac + if [ -z "$VALUE" ]; then + echo "Error: $KEY value is empty. Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2 + exit 1 + fi + if [ -z "$_mp_valid" ]; then + echo "Error: $KEY '$VALUE' not recognized. $_mp_rule Existing value left unchanged. fix: gstack-config unset $KEY restores the default ($_mp_doc)" >&2 + exit 1 + fi ;; + esac # The measure-loop controls bound paid spend: reject a typo instead of # coercing it, so a bad value never changes how much /ship may spend. case "$KEY" in @@ -893,7 +929,9 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \ + plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \ + model_smart_claude model_smart_openai; do gstack_config_select "$KEY" VALUE="$_gstack_cfg_value" SOURCE="default" @@ -916,7 +954,9 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend \ + plan_review_tier implementation_tier model_frontier_claude model_frontier_openai \ + model_smart_claude model_smart_openai; do printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")" done ;; diff --git a/bin/gstack-models b/bin/gstack-models new file mode 100755 index 000000000..ca6889494 --- /dev/null +++ b/bin/gstack-models @@ -0,0 +1,7 @@ +#!/usr/bin/env bun +import { modelsMain } from '../lib/model-policy-cli.ts'; + +const result = modelsMain(process.argv.slice(2)); +if (result.stdout) process.stdout.write(result.stdout); +if (result.stderr) process.stderr.write(result.stderr); +process.exit(result.code); diff --git a/claude-code/SKILL.md.tmpl b/claude-code/SKILL.md.tmpl index 0c5f18243..96a9a1b31 100644 --- a/claude-code/SKILL.md.tmpl +++ b/claude-code/SKILL.md.tmpl @@ -53,6 +53,9 @@ Set `GSTACK_CLAUDE_MODEL=` for an explicit override, including resumed consultations. If the user names a model, pass that value through this environment variable for every runner call. Without an override, retain Claude's configured model; harness routing never chooses a model family. +Only for an explicit `--role plan-review` (e.g. `/claude-code challenge --role plan-review`), +append `--role plan-review` to every runner call; the runner prints `CLAUDE_MODEL:` and +passes that one policy model (none in host mode). [Policy setup](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md). Do not infer authentication state from credential files or environment variables. Run the actual runner invocation in the host's normal execution context. On a diff --git a/codex/SKILL.md b/codex/SKILL.md index db1988de8..7e9f2ecf3 100644 --- a/codex/SKILL.md +++ b/codex/SKILL.md @@ -470,12 +470,16 @@ and every outside review run as a command, one subcommand per check. Model order: a model the user names for this request, `GSTACK_CODEX_MODEL`, Codex `config.toml` `model` (`review_model` first for `codex review`; honors `$CODEX_HOME`), then `gpt-6-astra`. Each call prints `CODEX_MODEL: (; source: ...)` first. -For a named model, add `--model ''` to every `select-model` and `probe-model` -call, including the probe below. An invalid or unavailable choice stops with a repair -message, never the default. +For a named model, add `--model ''` to every `select-model`, `probe-model` and +`role-ready` call, including the probe below. An invalid or unavailable choice stops +with a repair message, never the default; a named model also wins over a role. +Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`), +set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then +probes and dispatches the policy model within its own timeout. [Policy setup](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md). ```bash _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe +_CODEX_ROLE='' [ -x "$_CODEX_PROBE" ] || { echo "HELPER_UNAVAILABLE"; exit 1; } # GSTACK_ACTIVE_HOST names the harness, never the model. @@ -491,7 +495,7 @@ fi if ! "$_CODEX_PROBE" check-auth >/dev/null; then "$_CODEX_PROBE" log-event codex_auth_failed echo "AUTH_FAILED" -elif "$_CODEX_PROBE" check-sandbox; then # free; Linux only +elif "$_CODEX_PROBE" check-sandbox && [ -z "$_CODEX_ROLE" ]; then # free; Linux only "$_CODEX_PROBE" probe-model exec # ~10s round trip on first run, cached 1h fi "$_CODEX_PROBE" check-version # warns if known-bad, non-blocking @@ -847,11 +851,12 @@ If token count is not available, display: `Tokens: unknown` None of these is an auth or network failure, and the auth probe cannot catch them. Recovery, in order: 1. Read the `CODEX_MODEL:` line: it names the model and where it came from. - 2. Fix that source: name another model for this request, update - `GSTACK_CODEX_MODEL`, or change `model` (or `review_model`) in the Codex - `config.toml`. With none of these set, gstack uses `gpt-6-astra`; set any of - them to a model the account can use. - 3. If Codex printed `[notice.model_migrations]`, use that replacement model. + 2. Fix that source and relay its HINT/Repair lines. With `--role plan-review`, + tier pins outrank native settings; native `model`/`review_model` applies + only in `host` mode. Without a role, request, `GSTACK_CODEX_MODEL`, then + native settings choose the model, falling back to `gpt-6-astra`. + 3. If Codex printed `[notice.model_migrations]`, use that replacement for + no-role calls. For a role, offer an explicit override; do not silently substitute. Never present this as a model stall or a PASS — it is a fail-closed gate result. - **`VERDICT: unavailable`:** the shared validator found the run did not execute (for example `Codex's sandbox could not start here`). Relay its line and fix verbatim; it is diff --git a/codex/SKILL.md.tmpl b/codex/SKILL.md.tmpl index 61e5a440b..167bafc3f 100644 --- a/codex/SKILL.md.tmpl +++ b/codex/SKILL.md.tmpl @@ -70,19 +70,23 @@ and every outside review run as a command, one subcommand per check. Model order: a model the user names for this request, `GSTACK_CODEX_MODEL`, Codex `config.toml` `model` (`review_model` first for `codex review`; honors `$CODEX_HOME`), then `gpt-6-astra`. Each call prints `CODEX_MODEL: (; source: ...)` first. -For a named model, add `--model ''` to every `select-model` and `probe-model` -call, including the probe below. An invalid or unavailable choice stops with a repair -message, never the default. +For a named model, add `--model ''` to every `select-model`, `probe-model` and +`role-ready` call, including the probe below. An invalid or unavailable choice stops +with a repair message, never the default; a named model also wins over a role. +Only for an explicit `--role plan-review` (e.g. `/codex challenge --role plan-review`), +set `_CODEX_ROLE='plan-review'` below and in the mode block; the mode block then +probes and dispatches the policy model within its own timeout. [Policy setup](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md). ```bash _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe +_CODEX_ROLE='' [ -x "$_CODEX_PROBE" ] || { echo "HELPER_UNAVAILABLE"; exit 1; } {{OUTSIDE_SELF_GUARD:codex}} if ! "$_CODEX_PROBE" check-auth >/dev/null; then "$_CODEX_PROBE" log-event codex_auth_failed echo "AUTH_FAILED" -elif "$_CODEX_PROBE" check-sandbox; then # free; Linux only +elif "$_CODEX_PROBE" check-sandbox && [ -z "$_CODEX_ROLE" ]; then # free; Linux only "$_CODEX_PROBE" probe-model exec # ~10s round trip on first run, cached 1h fi "$_CODEX_PROBE" check-version # warns if known-bad, non-blocking @@ -313,11 +317,12 @@ If token count is not available, display: `Tokens: unknown` None of these is an auth or network failure, and the auth probe cannot catch them. Recovery, in order: 1. Read the `CODEX_MODEL:` line: it names the model and where it came from. - 2. Fix that source: name another model for this request, update - `GSTACK_CODEX_MODEL`, or change `model` (or `review_model`) in the Codex - `config.toml`. With none of these set, gstack uses `gpt-6-astra`; set any of - them to a model the account can use. - 3. If Codex printed `[notice.model_migrations]`, use that replacement model. + 2. Fix that source and relay its HINT/Repair lines. With `--role plan-review`, + tier pins outrank native settings; native `model`/`review_model` applies + only in `host` mode. Without a role, request, `GSTACK_CODEX_MODEL`, then + native settings choose the model, falling back to `gpt-6-astra`. + 3. If Codex printed `[notice.model_migrations]`, use that replacement for + no-role calls. For a role, offer an explicit override; do not silently substitute. Never present this as a model stall or a PASS — it is a fail-closed gate result. - **`VERDICT: unavailable`:** the shared validator found the run did not execute (for example `Codex's sandbox could not start here`). Relay its line and fix verbatim; it is diff --git a/codex/sections/challenge-mode.md b/codex/sections/challenge-mode.md index c27062e81..1e0294de1 100644 --- a/codex/sections/challenge-mode.md +++ b/codex/sections/challenge-mode.md @@ -39,6 +39,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`. ```bash _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; } +PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" +[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true) if [ -z "$PYTHON_CMD" ]; then echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2 @@ -49,11 +51,11 @@ fi [ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +_CODEX_ROLE='' +if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? +else _CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1; fi _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') -PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" -[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" -c "model=\"${_CODEX_SEL:?}\"" -c skills.include_instructions=false -c 'model_reasoning_effort="high"' -c 'web_search="cached"' --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c " import sys, json turn_completed_count = 0 diff --git a/codex/sections/challenge-mode.md.tmpl b/codex/sections/challenge-mode.md.tmpl index 8d9d30dee..49c3f1db5 100644 --- a/codex/sections/challenge-mode.md.tmpl +++ b/codex/sections/challenge-mode.md.tmpl @@ -30,6 +30,8 @@ If the user passed `--xhigh`, use `"xhigh"` instead of `"high"`. ```bash _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; } +PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" +[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } PYTHON_CMD=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true) if [ -z "$PYTHON_CMD" ]; then echo "ERROR: Python 3 is required to parse Codex JSON output. Install python3 or python and retry." >&2 @@ -39,9 +41,7 @@ fi # capture stderr to $TMPERR for auth error detection (was: 2>/dev/null). [ -n "${TMPERR:-}" ] || TMPERR=$(mktemp "$TMP_ROOT/codex-err-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } TMPRESP=$(mktemp "$TMP_ROOT/codex-resp-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } -{{CODEX_SELECT:exec}} -PROMPT_FILE="$(git rev-parse --show-toplevel 2>/dev/null || pwd)/.gstack/tmp/" -[ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } +{{CODEX_SELECT:exec:540}} "$_CODEX_PROBE" run-with-timeout 540 codex exec - -C "$_REPO_ROOT" -s "${_CODEX_SANDBOX_MODE:?}" {{CODEX_MODEL_CONFIG_FLAG}} -c 'model_reasoning_effort="high"' {{CODEX_WEB_SEARCH_FLAG}} --json -o "$TMPRESP" < "$PROMPT_FILE" 2>"$TMPERR" | tee "$TMPRESP.events" | PYTHONUNBUFFERED=1 "$PYTHON_CMD" -u -c " import sys, json turn_completed_count = 0 diff --git a/codex/sections/consult-mode.md b/codex/sections/consult-mode.md index 14f4a6650..a9b2e3c91 100644 --- a/codex/sections/consult-mode.md +++ b/codex/sections/consult-mode.md @@ -102,7 +102,9 @@ cd "$_REPO_ROOT" || exit 1 PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/" [ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +_CODEX_ROLE='' +if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+540)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? +else _CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1; fi _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') _SID="" diff --git a/codex/sections/consult-mode.md.tmpl b/codex/sections/consult-mode.md.tmpl index 81aa5c699..d080f833d 100644 --- a/codex/sections/consult-mode.md.tmpl +++ b/codex/sections/consult-mode.md.tmpl @@ -92,7 +92,7 @@ fi cd "$_REPO_ROOT" || exit 1 PROMPT_FILE="$_REPO_ROOT/.gstack/tmp/" [ -s "$PROMPT_FILE" ] || { echo "Not run: $PROMPT_FILE is missing or empty, so the prompt was never written. Write it, then run by hand: codex exec - -C $_REPO_ROOT < $PROMPT_FILE" >&2; exit 1; } -{{CODEX_SELECT:exec}} +{{CODEX_SELECT:exec:540}} _SID="" if [ "$_CODEX_MODE" = "resume" ]; then _SID=$(cat .context/codex-session-id 2>/dev/null) diff --git a/codex/sections/review-mode.md b/codex/sections/review-mode.md index ab4d41f4d..caf00ebad 100644 --- a/codex/sections/review-mode.md +++ b/codex/sections/review-mode.md @@ -48,7 +48,9 @@ _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" cd "$_REPO_ROOT" ~/.claude/skills/gstack/bin/gstack-review-log --start codex-review _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe -_CODEX_OUT=$("$_CODEX_PROBE" select-model review) || exit 1 +_CODEX_ROLE='' +if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+330)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready review) || exit $? +else _CODEX_OUT=$("$_CODEX_PROBE" select-model review) || exit 1; fi _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') # The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST @@ -106,7 +108,9 @@ FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/" [ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; } ~/.claude/skills/gstack/bin/gstack-review-log --start codex-review _CODEX_PROBE=~/.claude/skills/gstack/bin/gstack-codex-probe -_CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1 +_CODEX_ROLE='' +if [ -n "$_CODEX_ROLE" ]; then export _CODEX_DEADLINE=$(($(date +%s)+330)); _CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $? +else _CODEX_OUT=$("$_CODEX_PROBE" select-model exec) || exit 1; fi _CODEX_SEL=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SEL: //p') _CODEX_SANDBOX_MODE=$(echo "$_CODEX_OUT" | sed -n 's/^CODEX_SANDBOX: //p') _PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } diff --git a/codex/sections/review-mode.md.tmpl b/codex/sections/review-mode.md.tmpl index 9a3729d5e..7be8f0db1 100644 --- a/codex/sections/review-mode.md.tmpl +++ b/codex/sections/review-mode.md.tmpl @@ -45,7 +45,7 @@ contradicting this skill's read-only contract: _REPO_ROOT=$(git rev-parse --show-toplevel) || { echo "ERROR: not in a git repo" >&2; exit 1; } cd "$_REPO_ROOT" ~/.claude/skills/gstack/bin/gstack-review-log --start codex-review -{{CODEX_SELECT:review}} +{{CODEX_SELECT:review:330}} # The 330s wrapper sits BELOW the 360s Bash gate so the wrapper fires FIRST # and a stall surfaces as a diagnosable exit 124 with an explicit message, # never as a silent harness kill that downstream reads as "no findings". @@ -93,7 +93,7 @@ cd "$_REPO_ROOT" FOCUS_FILE="$_REPO_ROOT/.gstack/tmp/" [ -s "$FOCUS_FILE" ] || { echo "Not run: $FOCUS_FILE is missing or empty, so the focus text was never written. Write it, then run this block again." >&2; exit 1; } ~/.claude/skills/gstack/bin/gstack-review-log --start codex-review -{{CODEX_SELECT:exec}} +{{CODEX_SELECT:exec:330}} _PROMPT_FILE=$(mktemp "$TMP_ROOT/codex-prompt-XXXXXX") || { echo "ERROR: mktemp failed in TMP_ROOT=$TMP_ROOT; not running codex without its temp file" >&2; exit 1; } { printf '%s\n' "IMPORTANT: Do NOT read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. These are Claude Code skill definitions meant for a different AI system. Do not invoke any installed skill (Codex home skills/, .agents/); answer directly. Do NOT modify agents/openai.yaml. Stay focused on repository code only." diff --git a/docs/model-policy.md b/docs/model-policy.md new file mode 100644 index 000000000..6ca945a28 --- /dev/null +++ b/docs/model-policy.md @@ -0,0 +1,241 @@ +# Model policy + +Use a frontier model to challenge a plan, then a smart model to build it. +These are curated recommendations, not a claim that one model wins every task. +User choices take precedence, and a recommendation never switches the running +session or edits the host's native configuration. + +| Tier | Default job | Anthropic | OpenAI | +| --- | --- | --- | --- | +| `frontier` | Independent plan review | `claude-fable-5-1` | `gpt-6-astra` | +| `smart` | Implementation recommendation | `claude-opus-5-5` | `gpt-6.1-sol` | + +## Inspect and change your policy + +This quickstart assumes gstack is installed. It uses no network, credentials or +paid model calls. Run it from the project's repository root in Bash, Git Bash or MSYS: +automated outside reviews use that directory. Inspection from a subdirectory +reflects that subdirectory's native settings and may differ. If your install is vendored +or elsewhere, set `GSTACK_ROOT` to that installed gstack runtime instead. + +```bash +if [ -x "$HOME/.claude/skills/gstack/bin/gstack-models" ]; then + GSTACK_ROOT="$HOME/.claude/skills/gstack" +elif [ -x "${CODEX_HOME:-$HOME/.codex}/skills/gstack/bin/gstack-models" ]; then + GSTACK_ROOT="${CODEX_HOME:-$HOME/.codex}/skills/gstack" +else + printf '%s\n' 'Upgrade the gstack install for your host, then rerun this quickstart.' >&2 + exit 1 +fi +"$GSTACK_ROOT/bin/gstack-models" list +"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai +``` + +The output explains the selected model and which setting won. It also shows +catalog verification metadata. Offline selection does **not** prove that a CLI +is installed, that your account can use the model, or that a vendor's current +catalog has been checked. + +Choose the less expensive review tier, then inspect the effective result: + +```bash +"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier smart +"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider openai +``` + +Pin a provider's model within a tier, or reset a pin to the shipped default: + +```bash +"$GSTACK_ROOT/bin/gstack-config" set model_frontier_openai gpt-6-astra +"$GSTACK_ROOT/bin/gstack-config" set model_frontier_claude claude-fable-5-1 +"$GSTACK_ROOT/bin/gstack-config" unset model_frontier_openai +"$GSTACK_ROOT/bin/gstack-config" unset plan_review_tier +``` + +To keep using native CLI model settings for plan reviews: + +```bash +"$GSTACK_ROOT/bin/gstack-config" set plan_review_tier host +``` + +Settings apply to the next invocation without rerendering skills. An invocation +already being checked keeps its selected model through execution. Configuration +uses the existing [gstack state root](state-root.md), not a new per-repository +settings file. Invalid writes are rejected and leave the previous value intact; +use `unset`, not an empty value, to restore a default. + +A malformed known setting (for example `plan_review_tier = smart`) stops +selection rather than acting as an absent key. Correct or remove the named line +in the reported config file; `unset` only removes correctly spelled `key:` records. + +Codex plan-review readiness and dispatch share one deadline. Readiness is +budgeted half the remaining time, with the existing termination grace. A slow +preflight leaves less time for the review instead of extending the caller's +limit. An expired deadline starts no new command and means missing review +coverage, not a completed review. + +## Settings and precedence + +| Key | Default | Accepted value | +| --- | --- | --- | +| `plan_review_tier` | `frontier` | `frontier`, `smart`, `host` | +| `implementation_tier` | `smart` | `frontier`, `smart` | +| `model_frontier_claude` | Catalog's Anthropic frontier model | Explicit Anthropic model identifier | +| `model_frontier_openai` | Catalog's OpenAI frontier model | Explicit OpenAI model identifier | +| `model_smart_claude` | Catalog's Anthropic smart model | Explicit Anthropic model identifier | +| `model_smart_openai` | Catalog's OpenAI smart model | Explicit OpenAI model identifier | + +For plan reviews, a model named for that invocation wins over +`GSTACK_CLAUDE_MODEL` or `GSTACK_CODEX_MODEL`. That environment override wins over +the selected tier's configured model, which wins over the shipped catalog. +Generic coding-model settings do not silently select a cheaper plan reviewer; +choose `host` mode when that is the behavior you want. + +In `host` mode, Codex keeps its existing native-config resolution, including +`CODEX_HOME`, active profiles and the `exec` versus native `review` distinction. +Claude can delegate model selection to its CLI without forcing `--model`; the +inspector reports that delegation rather than inventing an exact model ID. +Actual returned model usage remains separate from the requested model. + +Implementation recommendations use a model explicitly requested for that job, +then the configured implementation tier and its provider override, then the +catalog. Outside-review environment overrides do not change implementation +recommendations. Known Claude Code and Codex hosts recommend their own provider; +an unknown or multi-provider host can show both choices. + +Custom endpoints and partner platforms need an explicitly compatible model or +`host` mode. Gstack does not guess that public API IDs work with Bedrock, Vertex, +Foundry or a custom Codex provider. A model identifier passing syntax validation +is not proof of account availability. See [provider recovery](troubleshooting.md#model-policy-provider). + +Detection is conservative, not a replacement for the CLI's settings engine. +Claude checks environment, user settings, project/shared/local settings, local +settings at the repository and worktree's main checkout, and local managed files. +Codex checks environment, user/active-profile settings and system configuration; +project-local Codex config is not inspected. +Any detected local custom-route signal requires an explicit model, even if +native trust or precedence rules might suppress it. Server-managed or MDM +policy and extra native CLI flags are not visible to this check; `--help` and +JSON output list these limits. Use an explicit compatible model for those cases, +or `host` mode for plan reviews. Implementation recommendations require an +explicit implementation model rather than delegating to a hidden native choice. + +## Which workflows use the tiers? + +The plan-review role applies to autoplan's CEO, design, DX and engineering +outside voices; standalone CEO, engineering, developer-experience and design +plan reviews; and the spec quality gate. Existing reasoning effort, access +restrictions, timeouts, review gates and `codex_reviews` consent remain in force. + +The host selects the opposing provider independently from the tier. Claude Code +uses the Codex wrapper; Codex uses the Claude Code wrapper. A model's name does +not establish which harness is running. + +Manual opposing-model wrappers can opt into `--role plan-review`. Ordinary +code reviews, live design reviews, design consultation, office-hours second +opinions and documentation reviews retain their existing model-selection +behavior. Historical no-role defaults, eval actors/judges, cheap helper models +and setup-time behavioral overlays do not follow this catalog. + +Before the first affected paid plan review after upgrade, gstack explains the +independent frontier default and the winning source, with `smart` and `host` +alternatives. Inspection does not mark that notice as seen. Disabled outside +reviews stay disabled; an unavailable model is reported, never silently replaced. + +## First completed outside review + +Policy setup and a completed review are different milestones. Install and +authenticate the opposing CLI using the [host setup instructions](../README.md#install--about-30-seconds) +before invoking a paid review. From Claude Code, use `/codex consult --role +plan-review`. From Codex, use `/claude-code consult --role plan-review` (or its +installed namespaced name). Ask it to challenge the plan and supply the full plan +text. Consult is prompt-driven and works before there is a repository diff; +Challenge mode is for reviewing code changes. + +Confirm the announced selection, the provider's actual result and the coverage +status. A resolved model, a successful availability probe or a native fallback +is not a completed outside review. An unusable explicit choice needs a repair +at the source named in the error; changing a lower-priority tier cannot override +an environment pin. See [selection recovery](troubleshooting.md#model-policy-selection). + +## Machine-readable inspection + +`gstack-models` with no arguments is `list`. Both `list` and `resolve` accept +`--json`. Plan-review resolution requires `--provider anthropic|openai`; +implementation resolution can omit the provider to return both recommendations. + +```bash +"$GSTACK_ROOT/bin/gstack-models" resolve --role plan-review --provider anthropic --json +"$GSTACK_ROOT/bin/gstack-models" resolve --role implementation --json +``` + +JSON output is one document with `schemaVersion: 1`, a `selections` array and +structured errors on failure. Human diagnostics go to stderr. Exit `0` means +valid resolution, including host delegation; `1` means configuration or +selection failure; `2` means invalid usage. Neither inspection nor resolution +writes notice markers, probes a model or fetches vendor sources. + +## Keeping shipped recommendations current + +The model-policy freshness workflow runs weekly and supports manual dispatch. +It checks official Anthropic and OpenAI recommendation and deprecation sources, +without provider credentials or paid model calls. Normal PR checks use offline +fixtures; a vendor outage does not make unrelated PRs fail. + +In GitHub Actions, select **Model-policy freshness** to run it manually. From a +gstack checkout, `bun scripts/model-policy-freshness.ts` performs the same public +source check without changing an issue; unlike `gstack-models`, it uses the +network. The workflow's guarded `--publish` path owns issue updates. Its JSON +report and job summary retain the source evidence. + +The report's primary status distinguishes `current`, `update-candidate` and +`unknown/source-unavailable`. Its separate `evidenceStatus` marks retained +successful evidence `stale` after thirty days; a failed fetch remains unknown +in the headline even when that retained evidence is stale. +Newer or newly recommended models are candidates, +not automatic replacements. A support promise such as “not sooner than” is not +a scheduled retirement, and an incidental model mention is not a new default. +This evidence covers public API recommendations, not partner-platform entitlement. + +One Actions-bot-owned tracking issue preserves the successful check's catalog/source/ +parser identity and any unresolved lifecycle findings. Healthy state remains +readable when the issue is closed. A failed or partial check cannot erase a known +retirement, borrow a success from an older catalog, or pretend its evidence is +fresh. A catalog rollback reopens historical findings for a model put back in +use. Only current default-branch runs may update that issue; copied markers in +user-owned issues are not trusted state. + +A documented retain decision preserves the current default, but does not make a +vendor mismatch `current`: the advisory issue stays open, or reopens, while that +candidate remains. Record retain rationales in issue comments, which the workflow +never edits, rather than racing a scheduled update to the issue body. + +Thirty-day staleness means an absence of trustworthy freshness evidence, not +that a model was released thirty days ago. An unchanged recommendation does not +require a date-only catalog commit or a monthly paid benchmark. The recorded +next-check due date helps maintenance, but a workflow that never runs cannot +alert on its own absence. + +### Maintainer update checklist + +1. Check the latest freshness run and its due date. Resolve missing, corrupt or + source-unavailable evidence before declaring the catalog current. +2. Verify the official recommendation and lifecycle evidence. Triage announced + retirement/removal within one working day, stale/unavailable evidence within + seven days and discretionary update candidates within thirty days. +3. Open a catalog change with the source URLs, exact model IDs and qualification + evidence, or record why the current default should remain. Never edit users' + personal pins or silently substitute another provider. +4. Confirm CLI model/profile support and tool/reasoning compatibility. Keep the + pinned test runtimes and benchmark rulers stable. A required CLI upgrade is a + separate reviewed change, not an automatic side effect of discovery. +5. Predeclare any budgeted baseline/candidate comparison using existing eval + machinery, fixed prompts/panels and unchanged thresholds. Keep every attempt, + measure useful and unsupported findings, latency and available usage, and do + not treat vendor marketing or a creation timestamp as quality evidence. +6. Pass the affected free checks and selected workflow evaluations, review the + change, and ship it through the normal release process. Only unpinned + role-bearing defaults follow the new shipped catalog. + +The workflow does not open an automatic model-upgrade PR, merge changes, switch +active sessions, alter benchmark models or schedule paid evaluations. diff --git a/docs/skills.md b/docs/skills.md index f027ab20a..2b9afcabb 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -1148,10 +1148,10 @@ This is my **second opinion mode**. `/codex` brings OpenAI Codex CLI to review the same diff independently. It is available on every harness except Codex itself. External harnesses install it as `/gstack-codex`. Compare its findings with the native review to distinguish corroborated findings from issues only one reviewer caught. -gstack-owned Codex calls default to `gpt-6-astra`, including resumed consult -sessions. Set `GSTACK_CODEX_MODEL=` to change the default, or name a -model in your request to override it for that invocation. Generated commands -pass the selection through `-c model=...`, overriding the CLI's configured model. +Without a role, gstack-owned Codex calls select a request-specific model, then +`GSTACK_CODEX_MODEL`, then Codex native settings, falling back to `gpt-6-astra`. +This also applies to resumed consultations. Generated commands pass the selection +through `-c model=...`, overriding the CLI's configured model for that call. Native review also sets `-c review_model=...` to that selection, overriding any separate review-model pin. @@ -1161,6 +1161,10 @@ review, challenge, and consult calls preserve Claude's configured model. sessions; a model named in your request takes precedence. Harness routing is independent of model selection. +An explicit `--role plan-review` uses the configured plan-review tier instead; +request and environment overrides still win, and `host` mode restores native +selection. See [model policy](model-policy.md) for pins, precedence and recovery. + ### Three modes **Review** — run `codex review` against the current diff. Codex reads every changed file, classifies findings by severity (P1 critical, P2 high, P3 medium), and returns a PASS/FAIL verdict. Any P1 finding = FAIL. The review is fully independent — Codex doesn't see Claude's review. @@ -1200,7 +1204,7 @@ Claude Code provides the outside reviewer when gstack runs in Codex. Other non-C **Review** supplies the branch diff for a read-only pass/fail review. **Challenge** asks Claude Code to find concrete failure cases in the same diff. **Consult** supports read-only repository exploration and resumes the session saved in `.context/claude-session-id`. Review and challenge receive context from the parent workflow and run without tools; consultation can read and search files. -The Claude Code CLI must be installed and authenticated. Its existing model configuration and `GSTACK_CLAUDE_BIN` / `GSTACK_CLAUDE_BIN_ARGS` executable overrides are honored. Errors, timeouts, and invalid responses report missing outside coverage instead of a clean review. Automatic reviews start fresh; consult session continuity is explicit. +The Claude Code CLI must be installed and authenticated. Without a role, its existing model configuration is honored; explicit `--role plan-review` follows the [model policy](model-policy.md). `GSTACK_CLAUDE_BIN` / `GSTACK_CLAUDE_BIN_ARGS` executable overrides remain in force. Errors, timeouts, and invalid responses report missing outside coverage instead of a clean review. Automatic reviews start fresh; consult session continuity is explicit. Outside-review routing follows the harness, independently of the configured model. Generic second-opinion requests choose `/claude-code` on Codex and `/codex` elsewhere; explicit provider requests keep that provider. The existing `codex_reviews` setting controls the selected automatic reviewer in workflows that already use that setting. Existing opt-in and skip controls still apply in office hours, design, and spec workflows. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2f28c7a53..4761a8775 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -34,6 +34,113 @@ no paid call: the Codex rows report the cached model check and its age, and --- +## Model policy + +The [model-policy guide](model-policy.md) explains the two tiers, all six +settings, precedence and the read-only inspection command. A resolved selection +is not proof that the requested provider or model can run. + + +### Model policy configuration is invalid or unreadable + +**Meaning.** The new policy path could not read a coherent configuration or a +configured value is invalid. An unreadable file is not treated as an absent +setting: doing that could silently select the premium default. + +**Fix.** Repair the file or key named by the error. `plan_review_tier` accepts +`frontier`, `smart` or `host`; `implementation_tier` accepts `frontier` or +`smart`. Use `gstack-config unset ` to restore a default, not an empty +value. A rejected `set` leaves the old value intact. Correct ownership/read +permissions rather than deleting an unrelated state directory. +For a malformed line such as `plan_review_tier = smart`, edit the reported file +and correct or remove that line first: `unset` only removes canonical `key:` +records and cannot repair a misspelled delimiter or quoted key. + +**Expected result.** `gstack-models list` or the corresponding `resolve` +command explains the winning source. No outside model call ran on this error. + + +### A custom provider needs an explicit model or host mode + +**Meaning.** Native settings or environment flags route the reviewer to a +custom endpoint or partner platform. A public API catalog identifier is not +necessarily a valid deployment identifier there. Malformed native provider +configuration also prevents gstack from assuming the public endpoint. + +**Fix.** Supply a compatible model for that request, set the provider's gstack +environment override or tier-model setting, or choose +`gstack-config set plan_review_tier host`. Keep credentials in the provider's +normal authentication mechanism, never in a model setting. If the error names +a malformed native configuration, fix that source first. + +**Expected result.** Inspection reports an explicit selection or honest +host-managed delegation. It does not certify account entitlement or send a +public default to an inferred custom endpoint. + + +### The selected review model could not be used + +**Meaning.** The requested model was rejected or the provider could not +complete the review. A model may be nonexistent, unavailable to the account, +retired, or behind a misconfigured endpoint; preserve the provider's evidence +rather than treating all of these as the same diagnosis. No successful outside +review is implied, and gstack does not substitute a different model. + +**Fix.** Change the source named in the error. For example, an environment +override beats a tier setting, so editing the tier cannot replace it. Unset or +correct that override, or explicitly name a supported model for the request. +Follow the existing auth, quota, sandbox or timeout repair when that is the +reported failure. Missing usage identity remains unknown, even if inspection +showed a requested ID. + +**Expected result.** The next invocation announces its model and source, +checks that same selection and reports the actual provider result. A cached +probe or a native fallback is not a completed outside review. + + +### Model-catalog freshness is unknown, stale or needs an update + +**Meaning.** Official evidence changed, a required source could not be +checked, or the retained evidence does not match this catalog and parser. +`unknown/source-unavailable` is not current. A later source failure must not +erase an earlier retirement finding or close its maintenance issue. + +**Fix.** Read the failed source and evidence details in the freshness workflow +report and tracking issue. Repair the source parser or publication access when +needed, then run the workflow manually against the default branch. If a model +was superseded or deprecated, follow the [maintainer checklist](model-policy.md#maintainer-update-checklist) +to qualify a replacement or record a supported mitigation. Do not clear the +issue just because a new release has not been adopted yet. + +**Expected result.** Complete source checks produce evidence bound to the +current catalog, while unresolved lifecycle findings remain visible. Personal +pins and unrelated PRs are unchanged. Check the workflow's last run date too: +a scheduled job that never runs cannot report its own absence. + +#### Repairing `recoveryRequired` state + +This flag deliberately keeps the issue open until a maintainer repairs the +record. Another successful fetch alone cannot certify corrupted history. + +1. Save the current issue body, including its retained recovery archive, and + download the latest trustworthy workflow report. Use its `report.state` as + the starting state; do not invent run IDs, catalog hashes or check dates. +2. Reconcile its `lifecycle` array with the issue's independent lifecycle block + and any retained archive. Preserve known retirements, deadlines and human + dispositions. A model currently in the catalog must not keep an old + `resolvedByCatalog` flag from a previous replacement. +3. In the issue editor, restore valid JSON between the `state:begin` and + `state:end` marker lines. Put that same lifecycle array between + `lifecycle:begin` and `lifecycle:end`. Restore the standalone ownership and + region markers, and leave human text outside the owned region intact. +4. Set `recoveryRequired` to `false` only after that reconciliation. If prior + successful evidence cannot be verified, set `lastSuccess` to `null` rather + than making up a fresh success. If lifecycle history cannot be recovered, + leave the flag set and the issue open for investigation. +5. Run **Model-policy freshness** against the default branch. Only complete, + current evidence with no unresolved findings can close the issue. A parsing + or source error leaves it open; do not clear history to manufacture a pass. + ## Outside reviews (Codex and Claude Code) Outside reviews send your diff, plan or question to a second AI provider. Their diff --git a/lib/claude-code-migration.ts b/lib/claude-code-migration.ts index 7b793116a..8ab1fc350 100644 --- a/lib/claude-code-migration.ts +++ b/lib/claude-code-migration.ts @@ -7,7 +7,12 @@ import { spawnSync } from 'node:child_process'; const OLD = 'gstack-claude'; const NEXT = 'gstack-claude-code'; const BANNER = ''; +export const FRESHNESS_REGION_BEGIN = ''; +export const FRESHNESS_REGION_END = ''; +const STATE_BEGIN = ''; +const STATE_END = ''; +const HISTORY_BEGIN = ''; +const HISTORY_END = ''; +export const FRESHNESS_ISSUE_AUTHOR = 'github-actions[bot]'; +export type TrackingIssue = { number: number; body: string; state: 'open' | 'closed'; author: string }; +export type FreshnessGitHub = { + defaultCatalog: (signal: AbortSignal) => Promise<{ defaultBranch: string; sourceSha256: string }>; + listIssues: (signal: AbortSignal) => Promise; + readIssue: (number: number, signal: AbortSignal) => Promise; + createIssue: (body: string, signal: AbortSignal) => Promise; + updateIssue: (number: number, update: { body: string; state: 'open' | 'closed' }, signal: AbortSignal) => Promise; +}; +export type StoredIssue = { state: FreshnessState | null; lifecycle: LifecycleFinding[]; recoveryRequired: boolean; problem: string | null }; +const KNOWN_MARKERS = [FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, STATE_BEGIN, STATE_END, HISTORY_BEGIN, HISTORY_END]; + +function markerLines(body: string, marker?: string): { start: number; end: number; text: string }[] { + return [...body.matchAll(/[^\r\n]+/g)].flatMap(line => { + const text = line[0]; + const matches = marker === undefined ? /^[\uFEFF \t]*(?:\n'; @@ -15,7 +16,7 @@ function fixture() { const root = path.join(dir, 'checkout'); const home = path.join(dir, 'home'); const codex = path.join(home, 'custom-codex', 'skills'); const kiro = path.join(home, '.kiro', 'skills'); - for (const rel of ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts']) put(path.join(root, rel), rel); + for (const rel of runtimeFiles) put(path.join(root, rel), rel); const oldRender = path.join(root, '.agents', 'skills', 'gstack-claude'); put(path.join(oldRender, 'SKILL.md'), skill('gstack-claude')); fs.mkdirSync(codex, { recursive: true }); fs.mkdirSync(kiro, { recursive: true }); @@ -40,6 +41,44 @@ function fixture() { } describe('Claude wrapper installed-name migration', () => { + test('copied policy tools and wrapper load without the source checkout', () => { + const f = fixture(); + try { + for (const rel of runtimeFiles) fs.copyFileSync(path.join(ROOT, rel), path.join(f.root, rel)); + put(path.join(f.codex, 'gstack-claude', 'SKILL.md'), skill('gstack-claude')); + expect(f.run({ copy: true })).toEqual({ migrated: 1, pending: [] }); + fs.renameSync(f.root, `${f.root}-unavailable`); + const runtime = path.join(f.codex, 'gstack'); + const env = { + ...process.env, HOME: f.home, USERPROFILE: f.home, + GSTACK_STATE_ROOT: path.join(f.home, 'state'), CODEX_HOME: path.dirname(f.codex), + CLAUDE_CONFIG_DIR: path.join(f.home, '.claude'), + GSTACK_CLAUDE_MODEL: '', GSTACK_CODEX_MODEL: '', ANTHROPIC_BASE_URL: '', OPENAI_BASE_URL: '', + CLAUDE_CODE_USE_BEDROCK: '', CLAUDE_CODE_USE_VERTEX: '', CLAUDE_CODE_USE_FOUNDRY: '', + BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0', + }; + const set = spawnSync('bash', [path.join(runtime, 'bin/gstack-config'), 'set', 'plan_review_tier', 'smart'], { + cwd: f.dir, env, encoding: 'utf8', timeout: 30_000, + }); + expect(set.status).toBe(0); + const inspect = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-models'), 'resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], { + cwd: f.dir, env, encoding: 'utf8', timeout: 30_000, + }); + expect(inspect.status).toBe(0); + expect(JSON.parse(inspect.stdout).selections[0].requestedModel).toBe('claude-opus-5-5'); + const wrapper = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-claude-code'), '--help'], { + cwd: f.dir, env, encoding: 'utf8', timeout: 30_000, + }); + expect(wrapper.status).toBe(1); + expect(JSON.parse(wrapper.stdout).error.code).toBe('arguments'); + const gate = spawnSync(process.execPath, ['-e', 'const {classifyOutsideReview} = await import(process.argv[1]); console.log(classifyOutsideReview({text:"",gate:"review",exit:1}).verdict)', path.join(runtime, 'lib/outside-review-result.ts')], { + cwd: f.dir, env, encoding: 'utf8', timeout: 30_000, + }); + expect(gate.status).toBe(0); + expect(gate.stdout.trim()).toBe('unavailable'); + } finally { fs.rmSync(f.dir, { recursive: true, force: true }); } + }); + test('migrates existing Codex and Kiro installs independently of the selected setup host', () => { const f = fixture(); try { diff --git a/test/claude-code-skill.test.ts b/test/claude-code-skill.test.ts index 92eda8582..61139c927 100644 --- a/test/claude-code-skill.test.ts +++ b/test/claude-code-skill.test.ts @@ -71,7 +71,7 @@ console.log(JSON.stringify({result:process.env.FAKE_TEXT || '[P1] Review found a }); afterAll(() => fs.rmSync(DIR, {recursive:true, force:true})); -function shell(mode: Mode, extra: NodeJS.ProcessEnv = {}, options: {resume?: boolean; runtime?: string; cwd?: string} = {}) { +function shell(mode: Mode, extra: NodeJS.ProcessEnv = {}, options: {resume?: boolean; runtime?: string; cwd?: string; planReview?: boolean} = {}) { fs.rmSync(CAPTURE, {force:true}); fs.writeFileSync(PROMPT, PROMPT_TEXT, {mode:0o600}); // These are precisely the literal substitutions the skill requests. Do not @@ -80,7 +80,8 @@ function shell(mode: Mode, extra: NodeJS.ProcessEnv = {}, options: {resume?: boo .replace("''", q(PROMPT)) .replace("''", q(options.runtime ?? RUNTIME)) .replace("''", q('main')) - .replace("''", q(options.resume ? 'resume' : 'fresh')); + .replace("''", q(options.resume ? 'resume' : 'fresh')) + .replaceAll('"$CLAUDE_RUNNER" --cwd', `"$CLAUDE_RUNNER"${options.planReview ? ' --role plan-review' : ''} --cwd`); return spawnSync('bash', ['-c',script], {cwd:options.cwd ?? REPO, env:{...ENV,...extra}, encoding:'utf8',timeout:10000}); } function captured() { return JSON.parse(fs.readFileSync(CAPTURE,'utf8')); } @@ -203,4 +204,22 @@ describe('complete generated Claude Code wrapper modes', () => { expectTempsCleaned(); } }); + + test('the documented plan-review consultation consumes a plan even without a repository diff', () => { + const clean = fs.mkdtempSync(path.join(DIR, 'plan-only-')); + git(['init', '-b', 'main'], clean); + fs.writeFileSync(path.join(clean, 'file.txt'), 'unchanged\n'); + git(['add', 'file.txt'], clean); + git(['commit', '-m', 'baseline'], clean); + const result = shell('consult', { HOME: clean, CLAUDE_CONFIG_DIR: path.join(clean, '.claude'), + GSTACK_STATE_ROOT: path.join(clean, 'policy'), GSTACK_CLAUDE_MODEL: '', FAKE_TEXT: 'The plan needs an overflow check.' }, + { cwd: clean, planReview: true }); + expect(result.status).toBe(0); + expect(result.stdout).toContain('The plan needs an overflow check.'); + const observed = JSON.parse(fs.readFileSync(CAPTURE, 'utf8')); + expect(observed.prompt).toBe(PROMPT_TEXT); + expect(observed.args).toContain('--model=claude-fable-5-1'); + expect(result.stdout).not.toContain('Nothing to review'); + expectTempsCleaned(); + }); }); diff --git a/test/codex-hardening.test.ts b/test/codex-hardening.test.ts index e42c916e7..174154f81 100644 --- a/test/codex-hardening.test.ts +++ b/test/codex-hardening.test.ts @@ -937,15 +937,30 @@ describe('codex broken-install detection (#2742)', () => { // hand-maintained copy that isn't resolver-generated — it must capture the // probe's exit code and route 2 to its own broken-install arm, or /autoplan // prints the wrong remedy for a broken binary. - test('autoplan preflight (tmpl + rendered) captures the probe exit and routes 2 to broken-install', () => { - for (const rel of ['autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md']) { - const raw = fs.readFileSync(path.join(ROOT, rel), 'utf-8'); - const src = rel.endsWith('.tmpl') ? raw.replace('{{OUTSIDE_PREFLIGHT:autoplan}}', RESOLVERS.OUTSIDE_PREFLIGHT({ host: 'claude', paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: rel }, ['autoplan'])) : raw; - expect(src).toContain('_CODEX_PO=$("$_CODEX_PROBE" probe-model $_CODEX_KIND); _CODEX_MP=$?'); - expect(src).toMatch(/_CODEX_MP" -eq 2/); - expect(src).toContain('binary cannot run'); - expect(src).not.toContain('elif ! "$_CODEX_PROBE" probe-model'); + test('autoplan (tmpl + rendered): availability pays for no probe; role-ready in the invocation probes, and its exit, 2 included, ends it', () => { + const ctx = { host: 'claude' as const, paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: 'autoplan/SKILL.md.tmpl' }; + const rendered = ['autoplan/SKILL.md', ...['ceo', 'design', 'dx', 'eng'].map(phase => `autoplan/sections/${phase}-phase.md`)] + .map(rel => fs.readFileSync(path.join(ROOT, rel), 'utf-8')).join('\n'); + for (const src of [RESOLVERS.OUTSIDE_PREFLIGHT(ctx, ['autoplan']) + RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan']), rendered]) { + expect(src).toContain('_CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $?'); + expect(src).not.toMatch(/probe-model|_gstack_codex_/); + expect(src).toContain('echo "CODEX_MODE: $_CODEX_MODE"'); } + const role = fs.readFileSync(PROBE, 'utf-8').match(/_gstack_codex_role_ready\(\) \{[\s\S]*?\n\}/)![0]; + expect(role).toContain('_gstack_codex_model_probe "$1" || return $?'); + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-role-broken-')); + try { + const bin = path.join(home, 'bin'); + fs.mkdirSync(bin); + fs.mkdirSync(path.join(home, '.codex')); + fs.writeFileSync(path.join(home, '.codex', 'auth.json'), '{}'); + fs.writeFileSync(path.join(bin, 'codex'), `#!/usr/bin/env bash\n[ "$1" = sandbox ] && exit 0\n[ "$1" = --version ] && { echo codex-cli 0.160.0; exit 0; }\necho 'Error: spawn /usr/lib/codex/vendor/codex ENOENT' >&2\nexit 1\n`, { mode: 0o755 }); + const r = spawnSync(PROBE, ['role-ready', 'exec'], { encoding: 'utf8', timeout: 20000, + env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: home, CODEX_HOME: path.join(home, '.codex'), GSTACK_HOME: path.join(home, 'state'), _TEL: 'off' } }); + expect(r.status).toBe(2); + expect(r.stdout).toBe(''); + expect(r.stderr).toContain('MODEL_UNUSABLE_INSTALL'); + } finally { fs.rmSync(home, { recursive: true, force: true }); } }); test('the preflight resolver routes exit 2 to broken_install', () => { diff --git a/test/codex-probe-cache.test.ts b/test/codex-probe-cache.test.ts new file mode 100644 index 000000000..002fdb921 --- /dev/null +++ b/test/codex-probe-cache.test.ts @@ -0,0 +1,297 @@ +/** + * Bounded multi-signature Codex probe cache (stubbed codex, no API spend): + * 16 entries, atomic publication, same-signature serialization with a + * bounded lock, abandoned-owner recovery, and unverified (never a duplicate + * paid probe) on lock timeout or unreadable cache. + */ +import { describe, expect, test } from 'bun:test'; +import { spawn, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe'); +const STUB = `#!/usr/bin/env bash +[ "$1" = sandbox ] && exit 0 +printf '%s\\n' "$*" >> "$STUB_LOG" +[ -n "\${STUB_SLEEP:-}" ] && sleep "$STUB_SLEEP" +case "\${STUB_MODE:-ok}" in + ok) echo OK; exit 0 ;; + model400) echo 'ERROR: {"type":"error","status":400,"error":{"message":"The model is not supported when using Codex with a ChatGPT account."}}' >&2; exit 1 ;; + quota) echo "ERROR: You've hit your usage limit. Try again at 3 AM." >&2; exit 1 ;; +esac +`; + +interface Fixture { home: string; codexHome: string; state: string; log: string; cache: string; locks: string; env: Record } + +function fixture(): Fixture { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-probe-cache-')); + const bin = path.join(home, 'bin'); + const codexHome = path.join(home, '.codex'); + const state = path.join(home, 'state'); + for (const dir of [bin, codexHome, state]) fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(bin, 'codex'), STUB, { mode: 0o755 }); + fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n'); + fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}'); + const log = path.join(home, 'stub.log'); + return { + home, codexHome, state, log, + cache: path.join(state, '.codex-model-probe'), + locks: path.join(state, '.codex-model-probe.locks'), + env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: home, CODEX_HOME: codexHome, GSTACK_HOME: state, STUB_LOG: log, _TEL: 'off' }, + }; +} + +function probe(f: Fixture, env: Record = {}, call = '_gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"') { + const r = spawnSync('bash', ['-c', `source "${PROBE}"\n${call}`], { env: { ...f.env, ...env }, encoding: 'utf8', timeout: 20000 }); + return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status }; +} + +function probeAsync(f: Fixture, env: Record): Promise { + return new Promise((resolve, reject) => { + const child = spawn('bash', ['-c', `source "${PROBE}"\n_gstack_codex_model_probe; echo "rc=$?"`], { env: { ...f.env, ...env } }); + let out = ''; + child.stdout.on('data', chunk => { out += chunk; }); + const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('probe hung')); }, 30000); + child.on('close', () => { clearTimeout(timer); resolve(out); }); + }); +} + +const calls = (f: Fixture) => (fs.existsSync(f.log) ? fs.readFileSync(f.log, 'utf8').split('\n').filter(Boolean).length : 0); +const entries = (f: Fixture) => fs.readFileSync(f.cache, 'utf8').split('\n').filter(line => /^MODEL_\S+ \d+ \d+$/.test(line)); +const model = (name: string) => ({ GSTACK_CODEX_MODEL: name }); +const withFixture = (body: (f: Fixture) => void | Promise) => async () => { + const f = fixture(); + try { await body(f); } finally { fs.rmSync(f.home, { recursive: true, force: true }); } +}; + +describe('bounded multi-signature probe cache', () => { + test('a write-lock holder releasing after a failed mkdir does not drop a negative entry', withFixture((f) => { + const lock = path.join(f.locks, 'write'); + const once = path.join(f.home, 'release-once'); + fs.mkdirSync(lock, { recursive: true }); + expect(fs.realpathSync(lock).startsWith(`${fs.realpathSync(f.home)}${path.sep}`)).toBe(true); + fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`); + fs.writeFileSync(once, 'armed'); + fs.writeFileSync(path.join(f.home, 'bin/mkdir'), `#!/usr/bin/env bash +if [ "$#" -eq 1 ] && [ "$1" = "$RACE_LOCK" ] && [ -f "$RACE_ONCE" ]; then + rm -f "$RACE_ONCE" "$RACE_LOCK/owner" + rmdir "$RACE_LOCK" + exit 1 +fi +exec ${JSON.stringify(Bun.which('mkdir'))} "$@" +`, { mode: 0o755 }); + const result = probe(f, { ...model('gpt-race'), STUB_MODE: 'quota', RACE_LOCK: lock, RACE_ONCE: once }); + expect(result.stdout).toContain('rc=4'); + expect(fs.existsSync(f.cache)).toBe(true); + expect(entries(f)).toHaveLength(1); + expect(entries(f)[0]).toStartWith('MODEL_QUOTA_EXHAUSTED '); + expect(probe(f, model('gpt-race')).stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)'); + expect(calls(f)).toBe(1); + })); + + test('A -> B -> A makes exactly two paid probes', withFixture((f) => { + expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK'); + expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK'); + const again = probe(f, model('gpt-a')); + expect(again.stdout).toContain('MODEL_OK (cached)'); + expect(calls(f)).toBe(2); + expect(entries(f)).toHaveLength(2); + expect(fs.existsSync(f.locks) ? fs.readdirSync(f.locks) : []).toEqual([]); + })); + + test('concurrent same-signature misses share one probe', withFixture(async (f) => { + const outputs = await Promise.all([0, 1, 2, 3].map(() => probeAsync(f, { ...model('gpt-same'), STUB_SLEEP: '1' }))); + expect(calls(f)).toBe(1); + for (const out of outputs) expect(out).toMatch(/MODEL_OK(?: \(cached\))?\nrc=0/); + expect(outputs.filter(out => out.includes('(cached)'))).toHaveLength(3); + expect(entries(f)).toHaveLength(1); + })); + + test('a slow writer keeps another signature published while its round trip ran; same-signature still replaces', withFixture(async (f) => { + const slow = probeAsync(f, { ...model('gpt-slow'), STUB_SLEEP: '3' }); + await Bun.sleep(1500); + expect(probe(f, { ...model('gpt-fast'), STUB_MODE: 'quota' }).stdout).toContain('rc=4'); + expect(await slow).toMatch(/MODEL_OK\nrc=0/); + expect(entries(f)).toHaveLength(2); + expect(probe(f, model('gpt-fast')).stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)'); + expect(probe(f, model('gpt-slow')).stdout).toContain('MODEL_OK (cached)'); + expect(calls(f)).toBe(2); + expect(probe(f, { ...model('gpt-fast'), GSTACK_CODEX_PROBE_RETRY: '1' }).stdout).toContain('MODEL_OK\n'); + expect(entries(f).map(line => line.split(' ')[0]).sort()).toEqual(['MODEL_OK', 'MODEL_OK']); + expect(probe(f, model('gpt-fast')).stdout).toContain('MODEL_OK (cached)'); + expect(calls(f)).toBe(3); + })); + + test('the 17th signature evicts the oldest entry; the file never exceeds 16', withFixture((f) => { + for (let i = 1; i <= 17; i++) expect(probe(f, model(`gpt-m${i}`)).stdout).toContain('MODEL_OK'); + expect(entries(f)).toHaveLength(16); + expect(probe(f, model('gpt-m17')).stdout).toContain('MODEL_OK (cached)'); + expect(probe(f, model('gpt-m2')).stdout).toContain('MODEL_OK (cached)'); + expect(calls(f)).toBe(17); + expect(probe(f, model('gpt-m1')).stdout).not.toContain('(cached)'); + expect(calls(f)).toBe(18); + expect(entries(f)).toHaveLength(16); + })); + + test('expired entries are pruned and never hit; TTLs and negative classifications are unchanged', withFixture((f) => { + probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' }); + probe(f, model('gpt-good')); + const now = Math.floor(Date.now() / 1000); + const lines = fs.readFileSync(f.cache, 'utf8').split('\n').filter(Boolean) + .map(line => line.replace(/^(MODEL_UNUSABLE) (\d+)/, (_, s) => `${s} ${now - 901}`)); + fs.writeFileSync(f.cache, `${lines.join('\n')}\n`); + expect(probe(f, { ...model('gpt-good') }).stdout).toContain('MODEL_OK (cached)'); + const bad = probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' }); + expect(bad.stdout).toContain('MODEL_UNUSABLE\n'); + expect(bad.stdout).toContain('rc=1'); + expect(calls(f)).toBe(3); + expect(probe(f, { ...model('gpt-bad'), STUB_MODE: 'ok' }).stdout).toContain('MODEL_UNUSABLE (cached)'); + expect(calls(f)).toBe(3); + const future = fs.readFileSync(f.cache, 'utf8').replace(/^MODEL_OK \d+/m, `MODEL_OK ${now + 999}`); + fs.writeFileSync(f.cache, future); + expect(probe(f, model('gpt-good')).stdout).not.toContain('(cached)'); + })); + + test('quota detail lines survive among other entries and are relayed on a hit', withFixture((f) => { + probe(f, model('gpt-x')); + expect(probe(f, { ...model('gpt-q'), STUB_MODE: 'quota' }).stdout).toContain('rc=4'); + probe(f, model('gpt-y')); + const hit = probe(f, model('gpt-q')); + expect(hit.stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)'); + expect(hit.stdout).toContain("ERROR: You've hit your usage limit. Try again at 3 AM."); + expect(hit.stdout).toContain('rc=4'); + expect(probe(f, { ...model('gpt-q'), GSTACK_CODEX_PROBE_RETRY: '1' }).stdout).toContain('MODEL_OK\n'); + expect(calls(f)).toBe(4); + expect(entries(f)).toHaveLength(3); + })); + + test('a pre-v2 one-entry cache migrates by TTL: fresh honored, expired re-probed, stray lines ignored', withFixture((f) => { + probe(f); + const [status, ts, sig] = entries(f)[0].split(' '); + expect(status).toBe('MODEL_OK'); + fs.writeFileSync(f.cache, `MODEL_OK ${ts} ${sig}\nan unprefixed legacy line\n`); + expect(probe(f).stdout).toContain('MODEL_OK (cached)'); + fs.writeFileSync(f.cache, `MODEL_OK ${Number(ts) - 3700} ${sig}\n`); + expect(probe(f).stdout).not.toContain('(cached)'); + expect(calls(f)).toBe(2); + })); + + test('an interrupted writer leaves a truncated line and temp file that never count as a hit', withFixture((f) => { + probe(f, model('gpt-a')); + fs.appendFileSync(f.cache, 'MODEL_OK 17'); + fs.writeFileSync(`${f.cache}.tmp.99999`, 'MODEL_OK 1 1\n'); + expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK (cached)'); + expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK\n'); + expect(fs.readFileSync(f.cache, 'utf8')).not.toContain('MODEL_OK 17\n'); + expect(entries(f)).toHaveLength(2); + })); + + test('config, auth and binary changes still invalidate a cached entry', withFixture((f) => { + probe(f); + const future = Date.now() / 1000 + 10; + fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future); + expect(probe(f).stdout).not.toContain('(cached)'); + fs.utimesSync(path.join(f.codexHome, 'auth.json'), future + 5, future + 5); + expect(probe(f).stdout).not.toContain('(cached)'); + expect(probe(f, { CODEX_API_KEY: 'k-test' }).stdout).not.toContain('(cached)'); + expect(calls(f)).toBe(4); + })); +}); + +describe('probe lock recovery and failure', () => { + const sigOf = (f: Fixture) => { + probe(f); + const sig = entries(f)[0].split(' ')[2]; + fs.rmSync(f.cache); + fs.rmSync(f.log); + return sig; + }; + + test('a dead owner on this host is recovered at once, without a duplicate probe', withFixture((f) => { + const sig = sigOf(f); + const lock = path.join(f.locks, sig); + fs.mkdirSync(lock, { recursive: true }); + const dead = spawnSync('bash', ['-c', 'echo $$'], { timeout: 5000 }).stdout.toString().trim(); + fs.writeFileSync(path.join(lock, 'owner'), `${dead} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`); + const r = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '3' }); + expect(r.stdout).toContain('MODEL_OK\n'); + expect(calls(f)).toBe(1); + expect(fs.existsSync(lock)).toBe(false); + })); + + test('a live owner past the stale bound is abandoned; within it the waiter is unverified without probing', withFixture((f) => { + const sig = sigOf(f); + const lock = path.join(f.locks, sig); + fs.mkdirSync(lock, { recursive: true }); + const now = Math.floor(Date.now() / 1000); + fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now}\n`); + const busy = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' }); + expect(busy.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (another probe of this model still holds its lock)'); + expect(busy.stdout).toContain('rc=0 state=inconclusive'); + expect(calls(f)).toBe(0); + fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now - 120}\n`); + expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n'); + expect(calls(f)).toBe(1); + })); + + test('a lock without its owner file is the creation window until it is 5 seconds old', withFixture((f) => { + const sig = sigOf(f); + const lock = path.join(f.locks, sig); + fs.mkdirSync(lock, { recursive: true }); + expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' }).stdout).toContain('MODEL_PROBE_INCONCLUSIVE'); + const old = Date.now() / 1000 - 30; + fs.utimesSync(lock, old, old); + expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n'); + expect(calls(f)).toBe(1); + })); + + test('an unreadable cache or uncreatable lock is unverified and launches no paid probe', withFixture((f) => { + fs.mkdirSync(f.cache); + const dir = probe(f); + expect(dir.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache'); + expect(dir.stdout).toContain('rc=0 state=inconclusive'); + fs.rmdirSync(f.cache); + fs.writeFileSync(f.cache, ''); + fs.chmodSync(f.cache, 0o000); + expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache'); + fs.chmodSync(f.cache, 0o644); + fs.rmSync(f.cache); + fs.writeFileSync(f.locks, 'not a directory'); + expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock'); + expect(calls(f)).toBe(0); + })); + + test('an owner record that cannot be written is lock I/O failure: unverified, no paid probe, no stray lock', withFixture((f) => { + fs.mkdirSync(f.locks, { recursive: true }); + const r = probe(f, {}, '_gstack_codex_select_model exec 2>/dev/null; umask 0777; _gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"'); + expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot record ownership of the probe lock'); + expect(r.stdout).toContain('rc=0 state=inconclusive'); + expect(calls(f)).toBe(0); + expect(fs.readdirSync(f.locks)).toEqual([]); + expect(fs.existsSync(f.cache)).toBe(false); + expect(probe(f).stdout).toContain('MODEL_OK\n'); + expect(calls(f)).toBe(1); + })); + + test('a cache write failure keeps the real probe result', withFixture((f) => { + fs.chmodSync(f.state, 0o500); + try { + const r = probe(f, { STUB_MODE: 'model400' }); + expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock'); + fs.chmodSync(f.state, 0o700); + fs.mkdirSync(f.locks); + fs.writeFileSync(path.join(f.locks, 'write'), 'blocks the write lock'); + const real = probe(f, { STUB_MODE: 'model400' }); + expect(real.stdout).toContain('MODEL_UNUSABLE\n'); + expect(real.stdout).toContain('rc=1'); + expect(fs.existsSync(f.cache)).toBe(false); + } finally { fs.chmodSync(f.state, 0o700); } + })); + + test('sourcing the probe still leaves the caller shell untouched', () => { + const r = spawnSync('bash', ['-c', `B=$(set -o; trap -p; echo "$IFS"); source "${PROBE}"; A=$(set -o; trap -p; echo "$IFS"); [ "$A" = "$B" ] && echo CLEAN`], { encoding: 'utf8', timeout: 5000 }); + expect(r.stdout.trim()).toBe('CLEAN'); + }); +}); diff --git a/test/codex-under-codex-detection.test.ts b/test/codex-under-codex-detection.test.ts index b2337750d..57cb3f5d7 100644 --- a/test/codex-under-codex-detection.test.ts +++ b/test/codex-under-codex-detection.test.ts @@ -90,7 +90,7 @@ describe('under-codex wiring renders (#2519)', () => { test('all three codexPreflight consumers render the probe', () => { for (const file of [ path.join(ROOT, 'ship', 'sections', 'adversarial.md'), - path.join(ROOT, 'plan-ceo-review', 'sections', 'review-sections.md'), + path.join(ROOT, 'review', 'sections', 'adversarial.md'), path.join(ROOT, 'document-release', 'sections', 'release-body.md'), ]) { const rendered = fs.readFileSync(file, 'utf-8'); diff --git a/test/docsync-fault-interface.test.ts b/test/docsync-fault-interface.test.ts index 7ca0ba476..8cbd23d61 100644 --- a/test/docsync-fault-interface.test.ts +++ b/test/docsync-fault-interface.test.ts @@ -7,7 +7,50 @@ import { DOC_PATH, docsCandidate, fixtureDocs, repoSnapshot } from './helpers/do import { DOCS_CHECKPOINT_MARKER, DOCS_SEEDED_AUDIT_ID, docsActorCommand, docsActorHook, docsActorSeeded, installDocsActor, seedDocsFirstAttempt, type DocsActorState } from './helpers/docsync-fault-actor'; import { docsActorVerdict } from './helpers/docsync-fault-eval'; import { extractDocsDispatch, parseDocsCompletion } from './helpers/docsync-contract'; -import { docsNativeInterface } from './helpers/docsync-observer'; +import { docsNativeInterface, docsWriteFailures, observeDocsWrites } from './helpers/docsync-observer'; + +test.skipIf(process.platform !== 'linux')('actor CLI validates owned state before creating even transient locks', async () => { + const fixture = fixtureDocs('current'); + let observer: Awaited> | undefined; + try { + const stateFile = installDocsActor(fixture, 'missing-asset'); + const stateBefore = fs.readFileSync(stateFile, 'utf8'); + const actor = path.join(import.meta.dir, 'helpers/docsync-fault-actor.ts'); + const malformed = path.join(fixture.repo, 'malformed-state.json'); + const foreign = path.join(fixture.repo, 'foreign-state.json'); + const otherRoot = path.join(fixture.home, 'other-owner'); + fs.mkdirSync(otherRoot); + fs.writeFileSync(malformed, '{'); + fs.writeFileSync(foreign, JSON.stringify({ root: otherRoot })); + const invalid = [['--help'], [], ['inspect'], ['hook'], + ['inspect', path.join(fixture.repo, 'missing-state.json')], ['inspect', malformed], ['inspect', foreign]]; + for (const args of invalid) { + observer = await observeDocsWrites(fixture); + const result = Bun.spawnSync([process.execPath, actor, ...args], { cwd: fixture.repo, + stdin: Buffer.from('{}'), stdout: 'pipe', stderr: 'pipe', timeout: 5000 }); + const observation = observer.stop(); observer = undefined; + expect(result.exitCode, JSON.stringify(args)).not.toBe(0); + expect(observation.complete).toBe(true); + expect(docsWriteFailures(observation, [])).toEqual([]); + expect(fs.readFileSync(stateFile, 'utf8')).toBe(stateBefore); + } + observer = await observeDocsWrites(fixture); + expect(docsActorCommand(stateFile, 'inspect').exit).toBe(0); + const valid = observer.stop(); observer = undefined; + expect(valid.complete).toBe(true); + expect(docsWriteFailures(valid, [])).toEqual([]); + + observer = await observeDocsWrites(fixture); + const forbidden = path.join(fixture.repo, 'unapproved.lock'); + fs.writeFileSync(forbidden, ''); fs.unlinkSync(forbidden); + const control = observer.stop(); observer = undefined; + expect(control.complete).toBe(true); + expect(docsWriteFailures(control, [])).toEqual(['forbidden docs write: unapproved.lock']); + } finally { + observer?.stop(); + fixture.clean(); + } +}); test('prepare copies the exact generated prompt and snapshots actual inputs without accepting an audit', () => { const fixture = fixtureDocs('current'); diff --git a/test/fixtures/model-policy-freshness.ts b/test/fixtures/model-policy-freshness.ts new file mode 100644 index 000000000..3ad72a585 --- /dev/null +++ b/test/fixtures/model-policy-freshness.ts @@ -0,0 +1,42 @@ +import { readFileSync } from 'node:fs'; +import { FRESHNESS_SOURCES, parseOfficialSource, sha256, type SourceId, type SourceResult } from '../../lib/model-policy-freshness-sources'; +import type { CatalogIdentity, FreshnessObservation } from '../../lib/model-policy-freshness'; + +export const freshnessCatalog: CatalogIdentity = { + sha256: sha256('catalog-v1'), sourceSha256: sha256('catalog-source-v1'), version: 1, + models: [ + { provider: 'anthropic', tier: 'frontier', modelId: 'claude-fable-5-1' }, + { provider: 'anthropic', tier: 'smart', modelId: 'claude-opus-5-5' }, + { provider: 'openai', tier: 'frontier', modelId: 'gpt-6-astra' }, + { provider: 'openai', tier: 'smart', modelId: 'gpt-6.1-sol' }, + ], +}; + +export function freshnessFixture(id: SourceId): string { + return readFileSync(new URL(`./model-policy-freshness/${id}.md`, import.meta.url), 'utf8'); +} + +export function freshnessObservation(id = '100', at = '2026-10-07T12:00:00.000Z', overrides: Partial> = {}): FreshnessObservation { + const sources: SourceResult[] = FRESHNESS_SOURCES.map(source => { + const text = overrides[source.id] === undefined ? freshnessFixture(source.id) : overrides[source.id]; + return text === null ? { id: source.id, error: 'http-503' } : { + id: source.id, receipt: { id: source.id, url: source.url, finalUrl: source.url, sha256: sha256(text!), bytes: Buffer.byteLength(text!), parsed: parseOfficialSource(source.id, text!) }, + }; + }); + return { catalog: structuredClone(freshnessCatalog), checkedAt: at, run: { id, attempt: 1, startedAt: new Date(Date.parse(at) - 1000).toISOString(), headSha: 'a'.repeat(40) }, sources }; +} + +export function retirementFixture(): string { + return freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027', 'claude-fable-5-1 | Deprecated | October 1, 2026 | November 1, 2026'); +} + +export function freshnessReplacementObservation(): FreshnessObservation { + const observation = freshnessObservation('101', '2026-10-08T12:00:00.000Z', { + 'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'), + 'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'), + }); + observation.catalog.models[0].modelId = 'claude-fable-5-2'; + observation.catalog.sha256 = 'b'.repeat(64); + observation.catalog.sourceSha256 = 'c'.repeat(64); + return observation; +} diff --git a/test/fixtures/model-policy-freshness/anthropic-lifecycle.md b/test/fixtures/model-policy-freshness/anthropic-lifecycle.md new file mode 100644 index 000000000..3814a510b --- /dev/null +++ b/test/fixtures/model-policy-freshness/anthropic-lifecycle.md @@ -0,0 +1,16 @@ +# Model deprecations + +Partner-operated platforms (Amazon Bedrock and Google Cloud) set their own retirement schedules. + +## Model status + +| API model name | Current state | Deprecated | Tentative retirement date | +| --- | --- | --- | --- | +| claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027 | +| claude-opus-5-5 | Active | N/A | Not sooner than September 22, 2027 | +| claude-mythos-preview | Deprecated | June 9, 2026 | To be announced | +| claude-opus-4-1-20250805 | Retired | June 5, 2026 | August 5, 2026 | + +## Deprecation history + +Amazon Bedrock partner discussion mentions `claude-fable-5-1` retirement; this is not a public API lifecycle row. diff --git a/test/fixtures/model-policy-freshness/anthropic-models.md b/test/fixtures/model-policy-freshness/anthropic-models.md new file mode 100644 index 000000000..042e41670 --- /dev/null +++ b/test/fixtures/model-policy-freshness/anthropic-models.md @@ -0,0 +1,17 @@ +# Models overview + +## Compare models + +If you're unsure which model to use, start with [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) for most workloads. Use [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) for demanding reasoning and long-horizon agentic work. + +| Feature | Claude Fable 5.1 | Claude Opus 5.5 | Claude Sonnet 5.5 | Claude Haiku 5.5 | +| --- | --- | --- | --- | --- | +| Model page | [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) | [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) | [Claude Sonnet 5.5](https://platform.claude.com/docs/en/models/sonnet-5-5/overview) | [Claude Haiku 5.5](https://platform.claude.com/docs/en/models/haiku-5-5/overview) | +| Claude API ID | `claude-fable-5-1` | `claude-opus-5-5` | `claude-sonnet-5-5` | `claude-haiku-5-5` | +| Amazon Bedrock ID | `anthropic.claude-fable-5-1` | `anthropic.claude-opus-5-5` | `anthropic.claude-sonnet-5-5` | `anthropic.claude-haiku-5-5` | + +* **Pricing:** prompt cache reads cost 10% of the base input price (2.5% on Claude Fable 5.1 and Claude Mythos 5.1, 5% on Claude Opus 5.5). See Pricing for per-platform pricing. Incidental ID: `claude-mythos-5-1`. + +## Using the Models API + +Legacy `claude-fable-5` and unrelated `claude-mythos-5-1` are not recommendations. diff --git a/test/fixtures/model-policy-freshness/openai-lifecycle.md b/test/fixtures/model-policy-freshness/openai-lifecycle.md new file mode 100644 index 000000000..4cffbb87f --- /dev/null +++ b/test/fixtures/model-policy-freshness/openai-lifecycle.md @@ -0,0 +1,24 @@ +# Deprecations + +This page lists all API deprecations, along with recommended replacements. + +## Upcoming deprecations + +### 2026-10-01: GPT-5.3-Codex + +| Shutdown date | Model / system | Recommended replacement | +| --- | --- | --- | +| Apr 1, 2027 | `gpt-5.3-codex` | `gpt-6-sol` | + +## Past deprecations + +| Shutdown date | Model snapshot | Substitute model | +| --- | --- | --- | +| Aug 5, 2026 | `gpt-old` | `gpt-6-astra` | +| 2026‑03‑26 | `gpt-unicode-date` | `gpt-6-astra` | +| 2026-02-12 | `gpt-iso-date` | `gpt-6-astra` | +| at earliest 2024-06-13 | `gpt-date-floor` | `gpt-6-astra` | +| 2026-09-24 | Videos API | --- | +| 2024-01-04 | `/v1/edits` | `/v1/chat/completions` | + +Incidental replacement text `gpt-6-astra` is not a deprecated model row. diff --git a/test/fixtures/model-policy-freshness/openai-models.md b/test/fixtures/model-policy-freshness/openai-models.md new file mode 100644 index 000000000..cedd4d17f --- /dev/null +++ b/test/fixtures/model-policy-freshness/openai-models.md @@ -0,0 +1,13 @@ +# Models + +## Featured models + +- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Start here for complex reasoning and coding. +- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Balance intelligence and cost. +- [GPT-6 Luna](/api/docs/models/gpt-6-luna.md): Optimize cost-sensitive, high-volume workloads. + +## Browse our full catalog of models + +- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Our most capable model for the most demanding work. +- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Near-Astra performance for complex work at a lower cost. +- [GPT-9 Preview](/api/docs/models/gpt-9-preview.md): Incidental full-catalog listing, not a featured recommendation. diff --git a/test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json b/test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json new file mode 100644 index 000000000..7fa9e2233 --- /dev/null +++ b/test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json @@ -0,0 +1,100 @@ +{ + "projection": "Literal public projection, not a full event replay: all 14 retained assistant messages, the ExitPlanMode use, final terminal viewport, and original complete request. Other 54 public tool events, raw/visible terminal history, and saved plan contents are omitted. Replay supplies synthetic command acknowledgment, owned session/path bindings and frame timing; it does not reconstruct unretained intermediate viewports.", + "source": { + "revision": "872e1542e42d40b3a95356d5284c6e4a3553c31f", + "ciRun": "37731666423", + "case": "plan-ceo-finding-floor", + "observationSha256": "78b7d36af226119c18de7b9e2d20b8bf28de87d3abd1006c4bd94fb6611a9aa8", + "viewportSha256": "6373752e484514929003f490a1c2ff8febc00fe1180f091ebf0cab6e8a7164ed", + "originalOutcome": "plan_ready", + "originalRequestSha256": "fb73beddb1f3a2634eabb15010e91f3d7232c7ce8d4da76ecef458136f93164a", + "cwd": "/home/runner/.cache/gstack-paid-shard-u6FAr4/tmp/gstack-plan-count-bowIFY" + }, + "originalRequest": "Proceed directly to the requested review; skip the optional /office-hours prerequisite.\n\nThis actor has already declined routing setup, cross-project recall and outside reviewers.\n\nPreserve the supplied product scope. For review-mode questions choose HOLD SCOPE (CEO), DX POLISH (DX), or the full BIG CHANGE review (Eng). Design: review all seven dimensions.\n\nPreserving scope does not approve the plan's premise, approach or any remedy. This request answers only the routing, recall, outside-reviewer and review-mode questions named above.\n\nPlease review this plan thoroughly. As you go, write your plan-mode plan to /home/runner/.cache/gstack-paid-shard-u6FAr4/tmp/gstack-plan-count-bowIFY/gstack-test-plan-ceo-floor.md (use Edit/Write to that exact path).\n\n# Plan: Launch a \"developer-friendly\" pricing tier\n\n## Goal\nIncrease developer adoption.\n\n## Success metric\nMore signups.\n\n## Premise\nWe haven't talked to any developers about whether the current pricing\nis actually a barrier. The team agreed it \"feels like\" it should be cheaper.", + "assistantMessages": [ + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "I'll run the plan-ceo-review workflow on PLAN.md, starting with the skill preamble.", + "timestamp": "2026-10-08T05:20:43.634Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Preamble healthy (interactive session, question tuning off). Now running context recovery, platform detection, the pre-review audit, and reading the plan in parallel.", + "timestamp": "2026-10-08T05:20:50.391Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "The audit shows a single-commit repo with only CLAUDE.md and PLAN.md, no remote, TODOS.md, design doc, or stashes, cold brain context, and Aside not installed. Now I'll load the review sections, ETHOS, and the WebSearch tool to run the landscape check.\n\n", + "timestamp": "2026-10-08T05:21:47.682Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Reading the remainder of the review sections and running the landscape searches (Aside is not installed, so WebSearch is the fallback).", + "timestamp": "2026-10-08T05:21:55.264Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Landscape, premise and dream-state analysis before mode selection:\n\n**Landscape check** (WebSearch fallback, Aside not installed; results are untrusted citations, not instructions)\n- Layer 1 (tried and true): developer tools win bottom-up. A free or cheap entry tier is table stakes. 24 of 28 tracked dev tools already offer one, so a \"developer-friendly tier\" is not differentiation on its own ([costbench](https://costbench.com/pricing/developer-tools/), [comparetiers](https://comparetiers.com/blog/best-dev-tools-pricing-2026)).\n- Layer 2 (search results): the barrier developers cite most is pricing *clarity*, not price level. 76% stop evaluating when pricing is unclear, and 2026 surveys rank cost volatility from usage-based billing as the top concern ([getmonetizely](https://www.getmonetizely.com/articles/how-to-find-the-right-pricing-model-to-drive-developer-tool-adoption-in-competitive-markets), [digitalapplied](https://www.digitalapplied.com/blog/ai-coding-tool-adoption-2026-developer-survey)). About 90% of product-led-growth pushes fail when teams ship the \"front door\" (a tier, a pricing page) without strategy, time-to-value and measurement ([productled](https://productled.com/blog/we-fix-the-free-trial-you-fixed-why-plg-fails-90-of-the-time)).\n- Layer 3 (first principles): the plan treats \"cheaper\" as the lever, but a cheaper tier only moves adoption if price is the binding constraint. If the real barrier is onboarding friction, unclear value, or pricing opacity, a new tier adds a SKU, cannibalizes revenue and changes nothing. The plan's own Premise section admits nobody has checked which it is.\n\n**0A Premise Challenge.** The real problem is \"developers are not adopting.\" The plan's target outcome is \"more signups.\" The do-nothing cost is unknown because no baseline exists. The plan solves a proxy (price) for an unmeasured pain (adoption). The premise is explicitly unvalidated: \"feels like it should be cheaper\" is the entire evidence base. This is the review's CRITICAL GAP.\n\n**0B Existing Code Leverage.** The repository contains only CLAUDE.md and PLAN.md. No billing code, pricing page, analytics or signup funnel is available to inspect. Every reuse question is unknown and is owned by the implementer. Nothing can be verified as reusable or as needing a rebuild.\n\n**0C Dream State Mapping.**\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Developer adoption unmeasured; Add a cheaper tier; Developers self-serve from\n no baseline, no interviews; hope signups rise; ---> first touch to paid, with a\n pricing barrier assumed ---> no gate, no kill criteria measured funnel, known barriers,\n pricing tested against evidence\n```\nThe plan moves sideways, not toward the ideal: it adds a pricing SKU without the measurement that the ideal depends on.\n\n**0D status.** The user chose HOLD SCOPE and stated that this request answers only routing, recall, outside-reviewer and review-mode questions. No approach choice is being auto-approved. The premise gap and three dependent gaps will be recorded as pending ledger rows with full decision briefs, not asked mid-run. Running the mode handoff now.", + "timestamp": "2026-10-08T05:23:35.992Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Mode: HOLD SCOPE; approved decisions: none. The mode was an explicit user choice in CLAUDE.md (\"choose HOLD SCOPE (CEO)\"), so no mode question was asked or logged. HOLD SCOPE applies because the user asked to preserve the supplied product scope: the review keeps the single deliverable (a developer-friendly pricing tier), proposes no expansions, and traces the gaps that stop the plan from meeting its own goal. No new approach decision was needed; the premise challenge becomes pending row D1 rather than an approved change.\n\nMode provenance: explicit user choice, HOLD SCOPE, no question log.\n\n**0G HOLD SCOPE checks**\n1. Complexity: the plan names zero files, classes or services. The threshold check cannot run. Recorded as unknown, owner: implementer.\n2. Minimum changes for the goal: the goal is developer adoption, not a tier. The minimum change is to find out whether price is the barrier. That is a repair required to meet the plan's own acceptance criterion, so under HOLD SCOPE rule 3 it belongs in scope, pending approval (D1). No existing scope item is proposed for deferral, so no defer/keep question arises.\n3. Invariants: the plan states one, \"more signups.\" It has no baseline, target, unit or window, so it cannot currently be met or failed. Repairing the metric is pending row D2.\n\n**0I Temporal Interrogation** (human / CC+gstack effort)\n```\n HOUR 1 (foundations): Which product? Which current tiers and prices? Who is a\n \"developer\" in the signup data? None is in the plan.\n HOUR 2-3 (core logic): What does the tier include and exclude? Price point?\n Grandfathering of existing customers? Billing-system changes?\n HOUR 4-5 (integration): Pricing page, checkout, entitlement checks, analytics\n events, revenue reporting. All unknown; no code to inspect.\n HOUR 6+ (polish/tests): Kill criteria, rollback for a public price change,\n cannibalization monitoring. The plan has none.\n```\nFeasibility blockers: premise unvalidated (D1), metric undefined (D2), rollback and grandfathering undefined (D3), launch observability undefined (D4). Sequence: D1 first, then D2, then D3 and D4 in parallel. Effort to resolve: human ~1 week of interviews and analysis / CC+gstack ~2 hours of plan work plus the interview calendar time, which AI does not compress.\n\nNow reading `review-sections.md` was completed above. Writing the initial plan file with Step 0 and the ledger, then proceeding through Sections 1 through 11.", + "timestamp": "2026-10-08T05:23:53.571Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "**Current scope.** Mode HOLD SCOPE (explicit user choice, no preference attribution). Governing rows: none approved. Pending: D1 premise gate, D2 success metric, D3 rollback and grandfathering, D4 launch observability. Accepted work: the single supplied deliverable, a developer-friendly pricing tier, as written. Deferred: none. Rejected: none.\n\nSection 1: Architecture Review\n\nThe plan names no components, so the architecture below is the minimum a pricing tier implies. Every box is unverified and owned by the implementer.\n\n```\n [Pricing page] --> [Checkout / billing provider] --> [Entitlement service]\n | | |\n v v v\n [Analytics: signup event] [Revenue reporting] [Feature gates in product]\n```\n\nFindings: no component boundaries, no data flows, no state machine for a subscription moving between tiers, no scaling analysis (not relevant at this depth), single point of failure is the billing provider, security boundary is the price-to-entitlement mapping, rollback posture undefined (D3). All route to existing pending rows. Decision gate: no new row, nothing to apply.\n\nSection 2: Error & Rescue Map. The plan defines no methods. Capability-level rows only: wrong entitlement granted for the tier, price shown that does not match price charged, grandfathered customers moved by mistake, signup events untagged by segment. Rescue status unknown for all. Cross-references D3 and D4. Gate: no new row.\n\nSection 3: Security & Threat Model. New attack surface: a cheaper tier is a target for abuse (multi-account signups to stay on the cheap tier, entitlement escalation by manipulating plan IDs at checkout). Likelihood Med, impact Med, plan does not mitigate. Payment data classification unchanged if the existing billing provider is reused. Gate: pending row D5 (abuse controls), recorded in the ledger as unresolved.\n\nSection 4: Data Flow & Interaction Edge Cases. Flow: visitor sees tier, clicks, checks out, gets entitlements, signup event recorded. Shadow paths: double-click at checkout creating two subscriptions, back button after payment, existing customer downgrading mid-cycle (proration), coupon stacking. None handled in the plan. Gate: routes to D3 and D5.\n\nSection 5: Code Quality Review. No code to assess. Finding: the plan is under-specified rather than under-engineered. Gate: no new row.\n\nAppending Sections 1 through 5 to the plan file.", + "timestamp": "2026-10-08T05:24:50.671Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Sections 1 through 5 saved. Continuing with Sections 6 through 11.\n\nSection 6: Test Review. New things: tier purchase flow, entitlement grant, grandfathering guard, developer-segment signup event, downgrade proration. No test exists in the plan for any. The 2am test: \"a customer charged the developer-tier price has exactly the developer-tier entitlements and no more.\" Hostile QA: tamper the plan ID at checkout. Chaos: billing-provider webhook delayed ten minutes after charge. Flakiness risk: anything hitting the provider's sandbox. No premise test exists either: nothing checks that the tier moved developer signups rather than all signups. Routes to D2 (metric assertion) and D1.\n\nSection 7: Performance. No new hot path. A pricing-page change is static. OK, no issues.\n\nSection 8: Observability. Nothing defined. Day-1 panels: developer-segment signups per day against baseline, tier conversion rate, revenue per new signup against pre-launch, downgrades from paid tiers. Alert: downgrade rate above a threshold (cannibalization). Runbook: pull the tier from the pricing page. Routes to D4.\n\nSection 9: Deployment & Rollout. A price change is public and customer-facing. Feature flag on the pricing page is the natural rollback switch. Risk window: customers who bought during the launch are on a tier that may be withdrawn. Routes to D3.\n\nSection 10: Long-Term Trajectory. Reversibility 2/5: the tier can be hidden quickly but customers on it are a durable commitment. Path dependency: an unvalidated tier becomes the anchor for all later pricing conversations. Debt: decision debt (no evidence), measurement debt (no baseline). Knowledge concentration: the plan has three sentences; a new engineer learns nothing from it.\n\nSection 11: Design & UX. The tier lands on the pricing page. States: loading, empty (tier unavailable), error (checkout failure), success, partial (charged, entitlements pending). None described. The research says pricing clarity is the barrier developers cite, so the tier's presentation is load-bearing, yet the plan says nothing about it. Recommend /plan-design-review once D1 resolves.", + "timestamp": "2026-10-08T05:25:36.159Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Sections 6 through 11 are saved. Since the outside voice was disabled per the user's choice, I'm recording that outcome while pulling prior review history and checking task-artifact prerequisites in parallel.\n\n", + "timestamp": "2026-10-08T05:26:29.928Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Outside voice is disabled, and under HOLD SCOPE all gaps (rows D1–D5) remain unresolved pending ledger items rather than backlog TODOs, so approval readiness is PASS with zero approved remedies. I'll now write the implementation tasks, completion summary, and terminal report to the plan file.\n\n", + "timestamp": "2026-10-08T05:26:57.831Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Report written. Running the read-back gate on the saved plan and writing the task artifact for autoplan in parallel.", + "timestamp": "2026-10-08T05:28:48.114Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Read-back checks pass—the review report ends correctly with the table, outside coverage, verdict, and final bullet intact, and the task artifact (2 tasks) is written. Next, I'll log the review history, check the dashboard, confirm the gate via Read, and leave a pointer at the host's plan path.\n\n", + "timestamp": "2026-10-08T05:29:12.128Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "Review and decision logs saved; dashboard shows NOT CLEARED since no Eng Review is on record, and the exit gate passed all five checks. Now running the brain cache refresh and telemetry.\n\n", + "timestamp": "2026-10-08T05:29:36.073Z" + }, + { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "text": "**STATUS: DONE_WITH_CONCERNS**\n\nThe review is complete and saved at `gstack-test-plan-ceo-floor.md` in the repo root, with a pointer at the host's plan path. Mode HOLD SCOPE, chosen by you. Nothing in the plan was approved or defaulted.\n\n**The verdict.** The plan cannot meet its own goal as written. It proposes a cheaper tier to raise developer adoption while admitting nobody has checked whether price is the barrier. Published research says developers cite pricing clarity and cost volatility far more than price level, and most tier launches that skip measurement fail. The success metric \"more signups\" has no baseline, segment, target or window, so the launch could be declared a success while developer adoption stays flat and paid customers downgrade.\n\n**Findings that matter**\n- **Premise unvalidated (CRITICAL GAP).** A week of developer interviews plus pricing-page funnel data would settle it. Pending D1, recommendation B: validate first and gate the launch.\n- **Metric unmeasurable (CRITICAL GAP).** Developer signups are not segmented anywhere in the plan. Pending D2, recommendation B: segmented metric with baseline and revenue guardrails.\n- **No rollback or grandfathering rule.** A public price change is a one-way door for whoever buys it. Pending D3.\n- **No dashboard, alert or runbook.** Pending D4.\n- **Checkout price integrity and multi-account abuse unmitigated.** Two High-impact threats. Pending D5.\n\nFailure-mode registry: 8 rows, 6 critical gaps. Reversibility 2 of 5. Performance: no issues.\n\n**Decisions left for you.** D1 through D5 each have a complete decision brief in the plan file under Unresolved Decisions. Only option-independent research tasks were emitted: T1 inventory the components a tier touches, T2 pull the signup baseline and define a developer signup. Both are in the autoplan task artifact.\n\n**Not done, by your instruction.** No outside reviewer ran (codex disabled, recorded). No /office-hours. No cross-project recall. No mid-run questions were asked. The host-side brain cache refresh reported 7 digest failures, which does not affect the review.\n\n**Next step.** Eng review is the required gate and has not run. Resolve D1 first, since D2 through D5 depend on whether the tier ships at all, then run `/plan-eng-review` in BIG CHANGE mode as you planned. Run `/plan-design-review` once the tier's contents are known.", + "timestamp": "2026-10-08T05:29:53.272Z" + } + ], + "exitPlanMode": { + "sessionId": "f79d8e58-6d3c-4c05-8861-e85dece9eda2", + "timestamp": "2026-10-08T05:29:53.383Z", + "toolUseId": "toolu_01SLfYKyirhjxLwqPYZrrrsC", + "kind": "use", + "name": "ExitPlanMode", + "input": { + "plan": "# CEO review of PLAN.md (pointer)\n\nThe user requested the plan-mode plan at an explicit path. The complete review, decision ledger, implementation tasks and GSTACK REVIEW REPORT live at:\n\n`/home/runner/.cache/gstack-paid-shard-u6FAr4/tmp/gstack-plan-count-bowIFY/gstack-test-plan-ceo-floor.md`\n\nSummary: HOLD SCOPE review of \"Launch a developer-friendly pricing tier\". Status: issues open. The premise (price is the adoption barrier) is unvalidated by the plan's own admission, the success metric \"more signups\" is unmeasurable, and 6 critical gaps were found. Five decisions (D1 premise gate, D2 metric, D3 rollback and grandfathering, D4 observability, D5 abuse controls) are recorded with full decision briefs and remain unresolved; nothing was approved or defaulted. Two option-independent research tasks (T1 component inventory, T2 signup baseline) are the only implementation tasks.\n", + "planFilePath": "/home/runner/.cache/gstack-paid-shard-u6FAr4/tmp/gstack-hermetic-672-gLL9sX/with-skills/.claude/plans/plan-md-optimized-llama.md" + }, + "messageId": "msg_011CfpDgXrRYhY3AJDQFHWWy", + "requestId": "req_011CfpDgXbJSJdKGTCEJNsmV" + }, + "viewport": " Decisions left for you. D1 through D5 each have a complete decision brief in the plan file under Unresolved Decisions.\n Only option-independent research tasks were emitted: T1 inventory the components a tier touches, T2 pull the signup\n baseline and define a developer signup. Both are in the autoplan task artifact.\n\n Not done, by your instruction. No outside reviewer ran (codex disabled, recorded). No /office-hours. No cross-project\n recall. No mid-run questions were asked. The host-side brain cache refresh reported 7 digest failures, which does not\n affect the review.\n\n Next step. Eng review is the required gate and has not run. Resolve D1 first, since D2 through D5 depend on whether\n the tier ships at all, then run /plan-eng-review in BIG CHANGE mode as you planned. Run /plan-design-review once the\n tier's contents are known.\n\n────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n Ready to code?\n\n Here is Claude's plan:\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n CEO review of PLAN.md (pointer)\n\n The user requested the plan-mode plan at an explicit path. The complete review, decision ledger, implementation tasks\n and GSTACK REVIEW REPORT live at:\n\n /home/runner/.cache/gstack-paid-shard-u6FAr4/tmp/gstack-plan-count-bowIFY/gstack-test-plan-ceo-floor.md\n\n Summary: HOLD SCOPE review of \"Launch a developer-friendly pricing tier\". Status: issues open. The premise (price is\n the adoption barrier) is unvalidated by the plan's own admission, the success metric \"more signups\" is unmeasurable,\n and 6 critical gaps were found. Five decisions (D1 premise gate, D2 metric, D3 rollback and grandfathering, D4\n observability, D5 abuse controls) are recorded with full decision briefs and remain unresolved; nothing was approved\n or defaulted. Two option-independent research tasks (T1 component inventory, T2 signup baseline) are the only\n implementation tasks.\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n\n────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n Claude has written up a plan and is ready to execute. Would you like to proceed?\n\n ❯ 1. Yes, and use auto mode\n 2. Yes, manually approve edits\n 3. Tell Claude what to change\n shift+tab to approve with this feedback\n" +} diff --git a/test/fixtures/qa-caller-read-probe-mtb1007a.json b/test/fixtures/qa-caller-read-probe-mtb1007a.json new file mode 100644 index 000000000..69da602bd --- /dev/null +++ b/test/fixtures/qa-caller-read-probe-mtb1007a.json @@ -0,0 +1,115 @@ +{ + "source": "ship-exploratory-late-input, mtb1007a, 2026-10-07; public synthetic fixture events only", + "events": [ + { + "type": "assistant", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "id": "msg_011CfoeQHondqjCMbvsEwUxZ", + "content": [ + { + "type": "tool_use", + "id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q", + "name": "Read", + "input": { + "file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/exploratory.md" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "content": [ + { + "tool_use_id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q", + "type": "tool_result", + "content": "1\t\n2\t\n3\t# Shared exploratory QA\n4\t\n5\tThe **caller** (/qa, /qa-only, /review or /ship) owns decisions, tests, fixes and publication. Discovery writes only reports/evidence\n6\tand owned fixture state; no workflows, framework installs or publication.\n7\t\n8\tComplete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation.\n9\t1. Read `sections/scope.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and select the surfaces.\n10\t2. Read the selected surface methods below in full.\n11\t\n12\t**Functional surfaces:**\n13\tRead `sections/system-functional.md` in full.\n14\t\n15\t**Browser surfaces only:**\n16\tRead `sections/qa-patterns.md` in full.\n17\t\n18\tMissing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks. Report QA setup blockers.\n19\t\n20\t## 1. Charter and preflight\n21\t\n22\tReuse resolved REPORT_DIR; otherwise own a fresh `.gstack/qa-reports` subdirectory.\n23\tWrite a **charter** per behavior: contract, risk, entrypoint, isolation, exit condition, source, commands and inputs. Save charters as Markdown in the report.\n24\t\n25\tFor /review and /ship, no plan/server is required.\n26\tStop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces).\n27\tExplicit plan checks and revalidation remain required beyond this smoke budget.\n28\tFor /qa and /qa-only:\n29\t- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900.\n30\t- Functional Full, Quick and Regression have no default total timer.\n31\tSet SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.\n32\tWithout a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.\n33\tStop when scoped contracts are tested or blocked.\n34\tClocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.\n35\tNames used below (quote every path):\n36\t- PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run.\n37\t- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.\n38\t- DEADLINE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-deadline`, the deadline guard: `start` creates DEADLINE_FILE, `status` prints `remainingMs` and `expired`, and `run` executes one command, stopping it at the deadline.\n39\t- EVIDENCE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence`, the functional evidence recorder: `capture` runs one command, stores its exit code, stdout and stderr under `PROBE_DIR/.qa-evidence/NNN/`, and prints status `complete`, `incomplete` or `sensitive`.\n40\t\n41\tStart once before baseline: `bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]` if bounded.\n42\tEARLIER_UTC = caller's absolute deadline, if set.\n43\tFunctional: `bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS`.\n44\tUnbounded: use `--timeout-ms MS` instead of `--deadline DEADLINE_FILE`.\n45\t--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of `PROBE_DIR/.qa-evidence/NNN/observation.json` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.\n46\tBounded browsers: `bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS`. No detached probes.\n47\tNever reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.\n48\t\n49\t## 2. Probe loop\n50\t\n51\tEach probe is one native command/interaction plus checks, excluding bookkeeping.\n52\tNever batch probes.\n53\t\n54\t1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.\n55\t2. **Decide whether another probe is needed.** If bounded, run `bun DEADLINE_TOOL status DEADLINE_FILE`.\n56\t If expired or no safe next probe remains, STOP exploration; write the report (\u00a74), not a checkpoint.\n57\t **Publish before probing.** Create (browser) or compose (functional) `exploration-NNN.json` in the probe directory, beside its deadline if bounded, with exactly four top-level fields:\n58\t observationCommand: last completed probe's full outer command, including guard.\n59\t observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text.\n60\t hypothesis: why nextCommand. nextCommand: exact command/request, guarded if bounded.\n61\t Preserve every safe program-JSON key/value and identity hash unchanged.\n62\t Withhold unsafe values, disclose limits and stop that chain.\n63\t Check fields before publication. No drafts/placeholders or invented safe-path redactions; corrections cannot repair published notes.\n64\t Functional: do not write this file; the next capture publishes it: `bun EVIDENCE_TOOL capture PROBE_DIR NNN --deadline DEADLINE_FILE --after PREV --hypothesis 'why' -- CMD` (PREV: the last complete capture's ID). EVIDENCE_TOOL supplies observed; never transcribe it.\n65\t Browser checkpoints use Write.\n66\t Wait for successful checkpoint publication before dispatch.\n67\t Never backfill or overwrite notes.\n68\t3. Run that exact probe; DEADLINE_TOOL enforces the deadline when bounded.\n69\t Report refusals as not-run; retain initial state/inputs/results. Repeat from step 2.\n70\t4. Replay the exact failing command/request from the same initial fixture state via steps 2\u20133 (same native command, fresh capture ID)\n71\t before repair, then minimize via those gates. Expiry leaves confirmation/minimization incomplete.\n72\t Another input or a regression test is not that replay.\n73\t5. After source/commands/fixtures change, re-review and return to step 2 for each affected revalidation (unproven=affected). Keep limits/notes; status requires fresh evidence.\n74\t\n75\t## 3. Parent handoff\n76\t\n77\t- **/qa:** parent owns severity, root-cause and Phase 8 regression gates before verified repair.\n78\t- **/review:** return before Fix-First; test_stub proposals require ASK approval.\n79\t- **Planning:** propose charters only; no execution.\n80\t\n81\tChoose the smallest native test: unit for logic, integration for state/requests; E2E only if smaller tests miss the journey, not automatically both.\n82\tMock only unrelated services.\n83\tNever freeze buggy output, weaken tests or delete valid red tests.\n84\t\n85\t## 4. Final report\n86\t\n87\tUse the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests.\n88\tWrite PROBE_DIR/annotations.json {evidence: [{capture, command, contract, expected, classification}], limits} (browser-only: evidence [], checkpoints in limits); before Markdown `bun EVIDENCE_TOOL materialize PROBE_DIR annotations.json` (fills observed/metadata; prints reportLinks; runs once per PROBE_DIR); you classify. Annotate every safe capture, including failures/replays: an omitted capture is withheld and keeps the verdict inconclusive. Classify a capture taken before an input change `superseded`; it closes when the same command reran on current inputs. Disclose withheld/incomplete evidence.\n89\tEvidence is invocation-local; /ship reruns once per invocation.\n90\tMissing prerequisites/expectations/observations, timeouts and refusal never pass.\n91\tPass requires all required current-input contracts to pass with no required remainder.\n92\tRequired failure leaves /review incomplete and /ship blocked unless the user explicitly accepts that named risk; noninteractive runs return blocked. Only nonbehavioral diffs may be not applicable (give a reason); prompts/templates are behavioral.\n93\t" + } + ] + } + }, + { + "type": "assistant", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "id": "msg_011CfoeQHondqjCMbvsEwUxZ", + "content": [ + { + "type": "tool_use", + "id": "toolu_01YbU4FzEXBhHBftjs4URWf5", + "name": "Bash", + "input": { + "command": "bun /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence capture /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports 001 --public --deadline /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports/deadline.json -- bun scripts/probe.ts 3", + "description": "Capture happy-path probe for input 3" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "content": [ + { + "tool_use_id": "toolu_01YbU4FzEXBhHBftjs4URWf5", + "type": "tool_result", + "content": "{\"id\":\"probe-ba7e52d4-ad96-4df9-97d0-6dd79e52f184\",\"charter\":\"happy\",\"input\":\"3\",\"snapshot\":\"936ff5b3b13172fb2357a120804c1cf5cac68d911c82cae5bce630f04dd62b57\",\"status\":\"pass\",\"stdout\":\"6\\n\",\"stderr\":\"\",\"exit\":0}\n\nQA_EVIDENCE {\"producer\":\"gstack-qa-evidence\",\"version\":1,\"action\":\"capture\",\"id\":\"001\",\"status\":\"complete\",\"sha256\":\"135ce1a8f804e3c6d953e0f284ab4e1e7fe5ded56f3a35706ea73f630876d0be\",\"exitCode\":0,\"signal\":null,\"publicOutput\":true,\"startedAt\":\"2026-10-07T22:13:30.620Z\",\"completedAt\":\"2026-10-07T22:13:30.675Z\",\"durationMs\":55,\"remainingMs\":182686,\"next\":\"Another probe requires a checkpoint anchored on capture 001: add --after 001 --hypothesis 'TEXT' before --. To stop exploring, run none.\"}", + "is_error": false + } + ] + }, + "tool_use_result": { + "interrupted": false + } + }, + { + "type": "assistant", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "id": "msg_011CfoeRZyA2j8av6NchLTmY", + "content": [ + { + "type": "tool_use", + "id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC", + "name": "Read", + "input": { + "file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/system-functional.md" + }, + "caller": { + "type": "direct" + } + } + ] + } + }, + { + "type": "user", + "parent_tool_use_id": null, + "session_id": "00ad7471-736b-4592-af5e-eace35c1d20d", + "message": { + "content": [ + { + "tool_use_id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC", + "type": "tool_result", + "content": "1\t\n2\t\n3\t# Functional QA with repository-native tools\n4\t\n5\tUse documented repository commands, CLI/API clients and job/queue tools, not a new\n6\tharness or browser substitution.\n7\t\n8\t## Functional modes\n9\t\n10\tFor /qa and /qa-only, within the selected scope:\n11\t- **Full** (default): cover every applicable documented contract below.\n12\t- **Quick** (`--quick`): check success and the highest-risk changed edge; mark other\n13\t contracts not run.\n14\t- **Regression** (`--regression `): before probes, read the supplied\n15\t functional report and linked replay evidence. A missing, unreadable or wrong-target\n16\t baseline blocks regression mode. A browser-only `baseline.json` is not a functional\n17\t baseline. Re-establish owned setup; replay prior failed probes against the documented\n18\t expectation, never recorded buggy output, then check changed adjacent behavior.\n19\t Preserve the prior report; report fixed, still failing and new findings separately.\n20\t Missing safe replay inputs block affected probes, never count as passes.\n21\t\n22\tMixed runs apply each surface's mode separately. /review and /ship retain their caller's\n23\tbounded smoke and explicit plan checks, not Full exploration.\n24\t\n25\t## Contract map\n26\t\n27\tRecord each contract/source, isolated setup, exact probe, expectation and outcome:\n28\tpass/fail/blocked/not run/inconclusive/not applicable (reason).\n29\t\n30\t| Contract | Observe |\n31\t|---|---|\n32\t| Successful execution | Expected return/output and final business effect, not just launch/acceptance |\n33\t| Invalid/missing input | Declared rejection, correct status and no forbidden state change |\n34\t| Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary |\n35\t| CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes |\n36\t| State transitions | Initial, intermediate and completed/failed states and their permitted transitions |\n37\t| Timeout/cancellation | Deadline, partial state, termination of owned work and recovery |\n38\t| Retry | Attempts/backoff/terminal state promised by the repository; no unbounded retry |\n39\t| Duplicates/idempotency | Repeated request/event and number of durable effects under the documented guarantee |\n40\t| Concurrency/order | Controlled competing operations in both relevant completion orders; final invariant |\n41\t| Partial-failure recovery | Interrupt after an effect, restart/replay, inspect completion/dead-letter state and duplicates |\n42\t\n43\tDo not impose universal exactly-once delivery. Separate acceptance, enqueue, processing,\n44\tretry/dead-letter and final effect; 2xx is not completion. Expected rejection/injected\n45\tfailure may pass; a missing service preventing execution blocks coverage.\n46\t\n47\t## Execute and retain evidence\n48\t\n49\t1. Apply the shared isolation/permission preflight. Verify cwd, command, environment\n50\t NAMES and safe reset; use synthetic data/credentials.\n51\t2. Follow the shared exploratory loop's order and written checkpoints.\n52\t For every probe, inspect initial/final durable state and retain exit/status and\n53\t stdout/stderr separately without masking failure.\n54\t3. On timeout, retain partial output/state and stop only owned work. Record setup errors\n55\t and untested contracts; never patch product code to hide missing prerequisites.\n56\t4. Record exact command or method/path/headers/body, setup/reset, expected contract/source,\n57\t observed output/state, revision/runtime, evidence paths and limits. Secrets are referenced\n58\t only by environment name. Disclose replay limits caused by redaction.\n59\t5. Use `templates/functional-report-template.md` relative to the installed QA SKILL.md.\n60\t Preserve evidence before owned cleanup and disclose leftovers. Return to the caller\n61\t without expanding discovery authority.\n62\t" + } + ] + } + } + ] +} diff --git a/test/helpers/docsync-fault-actor.ts b/test/helpers/docsync-fault-actor.ts index aa5a3dae2..74ad3d345 100644 --- a/test/helpers/docsync-fault-actor.ts +++ b/test/helpers/docsync-fault-actor.ts @@ -46,7 +46,7 @@ function save(file: string, state: DocsActorState) { } function locked(file: string, body: () => T): T { - const lock = file + '.lock'; + const lock = owned(load(file).root, file + '.lock'); const fd = fs.openSync(lock, 'wx', 0o600); try { return body(); } finally { fs.closeSync(fd); fs.unlinkSync(lock); } diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index ebb608670..781f6b729 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -124,6 +124,8 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'], 'test/fixtures/module-size-ratchet.json': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'], 'test/fixtures/module-size/**': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'], + 'test/fixtures/model-policy-freshness.ts': ['test/model-policy-freshness.test.ts', 'test/model-policy-workflow.test.ts'], + 'test/fixtures/model-policy-freshness/**': ['test/fixtures/model-policy-freshness.ts'], 'test/fixtures/native-auq-boxed-full-body-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], @@ -138,8 +140,10 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/parity-baseline-v1.81.0.0.json': ['test/skill-size-budget.test.ts'], 'test/fixtures/plan-count-design-questionless-report.md': ['test/plan-count-empty-review.test.ts'], 'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'], + 'test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json': ['test/plan-ceo-floor-interactive-boundary.test.ts'], 'test/fixtures/plan-floor-edit-wrap-37176835584.json': ['test/plan-floor-edit-wrap.test.ts'], 'test/fixtures/plan-format-kind-notes.json': ['test/plan-format-kind-note.test.ts'], + 'test/fixtures/qa-caller-read-probe-mtb1007a.json': ['test/qa-exploratory-callers.test.ts'], 'test/fixtures/plans/autoplan-dashboard.md': ['test/autoplan-init.test.ts', 'test/autoplan-preconfigured-onboarding-ar.test.ts', 'test/autoplan-snapshot.test.ts'], 'test/fixtures/plans/autoplan-focus-appearance-design.md': [], 'test/fixtures/plans/autoplan-focus-appearance.md': [], diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index 4ef0bcb7b..ca874247a 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -493,6 +493,7 @@ async function closePty(pty: PtyProcess, wallTimer: ReturnType probe.index <= read.resultIndex || !!probe.messageId && probe.messageId === read.messageId; + return resources.flatMap(([name, read]) => read && premature(read) ? [`probe preceded resource read: ${name}`] : []); +} + export interface CallerProbe { id: string; charter: string; @@ -358,10 +363,7 @@ export function validateCallerEvidence(input: { else { checkpointProbes.push({ command: String(tool.input.command), observed: probe, index: tool.index }); if (parentRead && (tool.index <= parentRead.resultIndex || tool.messageId && tool.messageId === parentRead.messageId)) errors.push(`probe preceded parent entrypoint: ${probe.id}`); - for (const id of ['exploratory', 'system-functional']) { - const resource = readOf(`/qa/sections/${id}.md`); - if (resource && (tool.index <= resource.resultIndex || tool.messageId && tool.messageId === resource.messageId)) errors.push(`probe preceded resource read: ${id}`); - } + errors.push(...callerProbeReadErrors(tool, ['exploratory', 'system-functional'].map(id => [id, readOf(`/qa/sections/${id}.md`)] as const))); if (input.requireGuardedSmoke) { const command = String(tool.input.command); const guarded = callerDeadlineCommand(command, deadline); diff --git a/test/model-catalog-isolation.test.ts b/test/model-catalog-isolation.test.ts new file mode 100644 index 000000000..1a649e8bc --- /dev/null +++ b/test/model-catalog-isolation.test.ts @@ -0,0 +1,82 @@ +import { expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +test('replacing the shipped catalog changes role choices, not eval rulers, overlays or no-role defaults', () => { + const source = path.resolve(import.meta.dir, '..'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-catalog-isolation-')); + try { + const files = [ + 'lib/model-catalog.ts', 'lib/model-policy.ts', 'lib/model-policy-notice.ts', 'lib/state-root.ts', + 'lib/eval-model.ts', 'lib/claude-code.ts', 'lib/claude-bin.ts', 'lib/claude-code-windows-job.ts', + 'lib/design-catalog.ts', 'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', + 'scripts/resolvers/constants.ts', 'scripts/resolvers/model-overlay.ts', + ...fs.readdirSync(path.join(source, 'hosts')).filter(file => file.endsWith('.ts')).map(file => `hosts/${file}`), + ...fs.readdirSync(path.join(source, 'model-overlays')).filter(file => file.endsWith('.md')).map(file => `model-overlays/${file}`), + ]; + for (const file of files) { + const target = path.join(root, file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(path.join(source, file), target); + } + const entry = path.join(root, 'inspect.ts'); + fs.writeFileSync(entry, ` +import { createHash } from 'node:crypto'; +import { resolvePlanReviewModel, resolveImplementationModels } from './lib/model-policy'; +import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from './lib/eval-model'; +import { CODEX_FRONTIER_MODEL } from './scripts/resolvers/constants'; +import { resolveCodexRuntimeModel, resolveCodexGenerationModel } from './scripts/resolve-codex-generation-model'; +import { ALL_MODEL_NAMES, resolveClaudeOverlay } from './scripts/models'; +import { generateModelOverlay } from './scripts/resolvers/model-overlay'; +import { ALL_HOST_CONFIGS } from './hosts/index'; +import { claudeCodeArgs } from './lib/claude-code'; +const scope = { env: process.env, cwd: process.cwd(), claudeManagedDir: './managed', codexSystemConfig: null }; +console.log(JSON.stringify({ + roles: ['anthropic', 'openai'].map(provider => resolvePlanReviewModel({ ...scope, provider }).requestedModel) + .concat(resolveImplementationModels(scope).map(selection => selection.requestedModel)), + independent: { + claudeRuler: CLAUDE_FRONTIER_EVAL_MODEL, openaiRuler: CODEX_FRONTIER_MODEL, + capture: resolveEvalModel('capture'), judge: resolveEvalModel('judge'), + codexExec: resolveCodexRuntimeModel({ kind: 'exec' }).model, + codexReview: resolveCodexRuntimeModel({ kind: 'review' }).model, + codexOverlay: resolveCodexGenerationModel().model, + claudeOverlay: resolveClaudeOverlay('claude-opus-5-5'), + claudeArgs: claudeCodeArgs({ access: 'none' }, { command: 'claude', argsPrefix: [] }, {}), + hostDefaults: ALL_HOST_CONFIGS.map(host => [host.name, host.defaultModel]), + overlays: ALL_MODEL_NAMES.map(model => [model, createHash('sha256').update(generateModelOverlay({ model })).digest('hex')]), + }, +})); +`); + const env = { + PATH: process.env.PATH ?? '', HOME: path.join(root, 'home'), + CODEX_HOME: path.join(root, 'home', '.codex'), GSTACK_STATE_ROOT: path.join(root, 'state'), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0', + }; + const inspect = () => { + const result = spawnSync(process.execPath, [entry], { cwd: root, env, encoding: 'utf-8', timeout: 30_000 }); + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + return JSON.parse(result.stdout); + }; + const before = inspect(); + expect(before.roles).toEqual(['claude-fable-5-1', 'gpt-6-astra', 'claude-opus-5-5', 'gpt-6.1-sol']); + expect(before.independent).toMatchObject({ + claudeRuler: 'claude-fable-5-1', openaiRuler: 'gpt-6-astra', capture: 'claude-fable-5-1', + judge: 'claude-fable-5-1', codexExec: 'gpt-6-astra', codexReview: 'gpt-6-astra', codexOverlay: 'gpt-6-astra', + }); + expect(before.independent.claudeArgs.some((arg: string) => arg.startsWith('--model'))).toBe(false); + const catalog = path.join(root, 'lib/model-catalog.ts'); + expect(fs.realpathSync(catalog).startsWith(`${fs.realpathSync(root)}${path.sep}`)).toBe(true); + const replacements = ['claude-sonnet-5', 'gpt-5.4', 'claude-opus-4-8', 'gpt-5.6-sol']; + let text = fs.readFileSync(catalog, 'utf-8'); + for (let i = 0; i < before.roles.length; i++) text = text.replace(`model: '${before.roles[i]}'`, `model: '${replacements[i]}'`); + fs.writeFileSync(catalog, text); + const after = inspect(); + expect(after.roles).toEqual(replacements); + expect(after.independent).toEqual(before.independent); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/model-policy-cli.test.ts b/test/model-policy-cli.test.ts new file mode 100644 index 000000000..db9996244 --- /dev/null +++ b/test/model-policy-cli.test.ts @@ -0,0 +1,323 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { modelCatalogSha256 } from '../lib/model-catalog'; +import { parseModelPolicyConfig, validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy'; +import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const MODELS = path.join(ROOT, 'bin', 'gstack-models'); +const CONFIG = path.join(ROOT, 'bin', 'gstack-config'); +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function fixture(config?: string) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack models cli ')); + roots.push(root); + const state = path.join(root, 'state'); + const home = path.join(root, 'home'); + fs.mkdirSync(state); + fs.mkdirSync(home); + if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config); + return { root, state, home, env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } as Record }; +} + +function models(args: string[], env: Record, cwd = path.dirname(env.HOME)) { + const result = spawnSync(process.execPath, [MODELS, ...args], { encoding: 'utf-8', env, cwd, timeout: 30_000 }); + return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +function config(args: string[], env: Record) { + const result = spawnSync('bash', [CONFIG, ...args], { encoding: 'utf-8', env, timeout: 30_000 }); + return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +function json(args: string[], env: Record, cwd?: string) { + const result = models([...args, '--json'], env, cwd); + expect(result.stderr).toBe(''); + return { code: result.code, doc: JSON.parse(result.stdout) }; +} + +function tree(dir: string): string[] { + return fs.readdirSync(dir, { recursive: true }).map(String).sort(); +} + +describe('gstack-models inspector', () => { + test('--help explains roles, values, offline behavior, delegation and exits', () => { + const result = models(['--help'], fixture().env); + expect(result.code).toBe(0); + for (const text of ['plan-review', 'implementation', 'frontier | smart | host', 'frontier | smart ', 'offline', 'no paid model call', 'host-controlled', 'Exit: 0', '2 invalid usage', 'schemaVersion 1']) { + expect(result.stdout).toContain(text); + } + }); + + test('list is the default and emits one JSON document with both roles and tier entries', () => { + const { env, state } = fixture(); + const human = models([], env); + expect(human.code).toBe(0); + expect(human.stdout).toContain('plan-review via openai: gpt-6-astra'); + expect(human.stdout).toContain('implementation via anthropic: claude-opus-5-5'); + expect(human.stdout).toContain('gstack-config set plan_review_tier smart'); + expect(human.stderr).toBe(''); + expect(models(['list'], env).stdout).toBe(human.stdout); + + const { code, doc } = json([], env); + expect(code).toBe(0); + expect(doc).toMatchObject({ schemaVersion: 1, command: 'list', ok: true, offline: true, configPath: path.join(state, 'config.yaml'), errors: [] }); + expect(doc.catalog).toMatchObject({ version: 1, sha256: modelCatalogSha256() }); + expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/anthropic', 'plan-review/openai', 'implementation/anthropic', 'implementation/openai']); + expect(doc.tiers).toHaveLength(4); + expect(doc.settings.plan_review_tier).toEqual({ value: null, effective: 'frontier', origin: 'default' }); + expect(doc.settings.model_smart_openai).toEqual({ value: null, effective: 'gpt-6.1-sol', origin: 'default' }); + }); + + test('resolve plan-review returns one tagged selection with its source', () => { + const { env } = fixture('model_frontier_openai: gpt-pinned\n'); + const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CLAUDE_MODEL: 'claude-ignored' }); + expect(code).toBe(0); + expect(doc.selections).toHaveLength(1); + expect(doc.selections[0]).toMatchObject({ status: 'selected', role: 'plan-review', provider: 'openai', tier: 'frontier', requestedModel: 'gpt-pinned', source: { kind: 'config', key: 'model_frontier_openai' } }); + const envWins = json(['resolve', '--role=plan-review', '--provider=openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt-env' }); + expect(envWins.doc.selections[0]).toMatchObject({ requestedModel: 'gpt-env', source: { kind: 'env', name: 'GSTACK_CODEX_MODEL' } }); + }); + + test('every tier entry includes executable pin and reset commands', () => { + const { env, root } = fixture(); + const install = path.join(root, "installed gstack's runtime"); + for (const file of CLAUDE_CODE_RUNTIME_FILES) { + const target = path.join(install, file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(path.join(ROOT, file), target); + } + const listed = spawnSync(process.execPath, [path.join(install, 'bin/gstack-models'), 'list'], { cwd: root, env, encoding: 'utf-8', timeout: 30_000 }); + expect(listed.status).toBe(0); + const lines = listed.stdout.split('\n').map(line => line.trim()); + const { doc } = json([], env); + for (const entry of doc.tiers) { + const pin = lines.find(line => line.includes(` set ${entry.overrideKey} `)); + const reset = lines.find(line => line.endsWith(` unset ${entry.overrideKey}`)); + expect(pin).toBeDefined(); + expect(reset).toBeDefined(); + for (const command of [pin!, reset!]) { + const result = spawnSync('bash', ['-c', command], { cwd: root, env, encoding: 'utf-8', timeout: 30_000 }); + expect(result.status).toBe(0); + const setting = json([], env).doc.settings[entry.overrideKey]; + expect(setting.origin).toBe(command === pin ? 'config' : 'default'); + expect(setting.effective).toBe(entry.catalogModel); + } + } + }); + + test('resolve implementation returns both providers unless one is named', () => { + const { env } = fixture('implementation_tier: frontier\n'); + const both = json(['resolve', '--role', 'implementation'], env); + expect(both.code).toBe(0); + expect(both.doc.selections.map((s: { provider: string; requestedModel: string }) => [s.provider, s.requestedModel])).toEqual([['anthropic', 'claude-fable-5-1'], ['openai', 'gpt-6-astra']]); + const one = json(['resolve', '--role', 'implementation', '--provider', 'openai'], env); + expect(one.doc.selections).toHaveLength(1); + expect(one.doc.selections[0].provider).toBe('openai'); + }); + + test('host mode is a valid delegated resolution with a null model', () => { + const { env } = fixture('plan_review_tier: host\n'); + const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env); + expect(code).toBe(0); + expect(doc.selections[0]).toMatchObject({ status: 'delegated-host', requestedModel: null, source: { kind: 'host', reason: 'host-mode' } }); + const human = models(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env); + expect(human.code).toBe(0); + expect(human.stdout).toContain('host-controlled'); + }); + + test.each([ + [['bogus'], "unknown command 'bogus'"], + [['resolve'], 'resolve needs --role'], + [['resolve', '--role', 'coding'], "unknown role 'coding'"], + [['resolve', '--role', 'plan-review'], 'needs --provider'], + [['resolve', '--role', 'plan-review', '--provider', 'claude'], "unknown provider 'claude'"], + [['resolve', '--role', 'implementation', '--provider', 'OpenAI'], "unknown provider 'OpenAI'"], + [['resolve', '--role', 'plan-review', '--role', 'implementation', '--provider', 'openai'], '--role given twice'], + [['resolve', '--role'], '--role needs a value'], + [['list', '--provider', 'openai'], 'unknown argument'], + ])('usage error %p exits 2', (args, message) => { + const { env } = fixture(); + const human = models(args, env); + expect(human.code).toBe(2); + expect(human.stdout).toBe(''); + expect(human.stderr).toContain(message); + const { code, doc } = json(args, env); + expect(code).toBe(2); + expect(doc).toMatchObject({ schemaVersion: 1, ok: false, selections: [] }); + expect(doc.errors[0]).toMatchObject({ reason: 'usage' }); + expect(doc.errors[0].problem).toContain(message); + }); + + test('invalid config exits 1 with structured, source-aware repair', () => { + const { env } = fixture('plan_review_tier: fronteir\n'); + const { code, doc } = json([], env); + expect(code).toBe(1); + expect(doc).toMatchObject({ ok: false, selections: [], settings: null }); + expect(doc.errors[0]).toMatchObject({ reason: 'invalid_plan_review_tier', key: 'plan_review_tier' }); + expect(doc.errors[0].repair).toContain('gstack-config unset plan_review_tier'); + expect(doc.errors[0].docs).toContain('#model-policy-config'); + const human = models(['resolve', '--role', 'implementation'], env); + expect(human.code).toBe(1); + expect(human.stdout).toBe(''); + expect(human.stderr).toContain('fix: gstack-config unset plan_review_tier'); + }); + + test('an unusable environment override points at the env var', () => { + const { env } = fixture(); + const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt 6' }); + expect(code).toBe(1); + expect(doc.errors[0]).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', role: 'plan-review', provider: 'openai' }); + expect(doc.errors[0].repair[0]).toBe('unset GSTACK_CODEX_MODEL'); + }); + + test('custom routing fails only the affected selections', () => { + const { env } = fixture(); + const { code, doc } = json([], { ...env, CLAUDE_CODE_USE_BEDROCK: '1' }); + expect(code).toBe(1); + expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/openai', 'implementation/openai']); + expect(doc.errors.map((e: { reason: string; role: string }) => `${e.reason}/${e.role}`)).toEqual(['custom_provider_requires_model/plan-review', 'custom_provider_requires_model/implementation']); + }); + + test('project settings in the working directory are custom routing; detection limits are published', () => { + const { env, root } = fixture(); + const project = path.join(root, 'project'); + fs.mkdirSync(path.join(project, '.claude'), { recursive: true }); + fs.writeFileSync(path.join(project, '.claude', 'settings.local.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } })); + const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env, project); + expect(code).toBe(1); + expect(doc.errors[0]).toMatchObject({ reason: 'custom_provider_requires_model', provider: 'anthropic' }); + expect(doc.providerDetection.unscanned.anthropic.join(' ')).toContain('server-managed'); + expect(json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env).code).toBe(0); + expect(models(['--help'], env).stdout).toContain('cannot see server-managed or MDM policy'); + }); + + test.each(['[]', 'null', '"unsupported"', 'true', '42'])('unsupported native settings root %s fails closed without writes', (contents) => { + const { root, home, env } = fixture(); + const directory = path.join(home, '.claude'); + fs.mkdirSync(directory); + const settings = path.join(directory, 'settings.json'); + fs.writeFileSync(settings, '{}'); + const args = ['resolve', '--role', 'plan-review', '--provider', 'anthropic']; + expect(json(args, env).code).toBe(0); + fs.writeFileSync(settings, contents); + const before = tree(root); + const result = json(args, env); + expect(result.code).toBe(1); + expect(result.doc.selections).toEqual([]); + expect(result.doc.errors[0]).toMatchObject({ reason: 'native_provider_unresolved', key: settings }); + expect(result.doc.errors[0].repair).toContain(`fix the settings object in ${settings}`); + expect(tree(root)).toEqual(before); + expect(json(args, { ...env, GSTACK_CLAUDE_MODEL: 'claude-explicit' }).doc.selections[0].requestedModel).toBe('claude-explicit'); + }); + + test('inspection never writes to the state root or home', () => { + const { root, env } = fixture('plan_review_tier: smart\n'); + const before = tree(root); + models([], env); + models(['--json'], env); + models(['resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], env); + models(['resolve', '--role', 'implementation'], env); + expect(tree(root)).toEqual(before); + }); +}); + +describe('gstack-config model policy keys', () => { + test('the annotated header has valid uncommentable tier settings and no empty pin examples', () => { + const { env, state } = fixture(); + expect(config(['set', 'telemetry', 'off'], env).code).toBe(0); + const text = fs.readFileSync(path.join(state, 'config.yaml'), 'utf8'); + const tiers = text.split('\n').filter(line => /^# (plan_review_tier|implementation_tier):/.test(line)).map(line => line.slice(2)); + expect(tiers).toHaveLength(2); + const parsed = parseModelPolicyConfig(tiers.join('\n'), '/example/config.yaml'); + expect(parsed.planReviewTier.value).toBe('frontier'); + expect(parsed.implementationTier.value).toBe('smart'); + expect(text).not.toMatch(/^# model_(frontier|smart)_(claude|openai):\s*(?:#.*)?$/m); + }); + + test('Anthropic pins preserve the 512-character contract with a POSIX-bounded regex engine', () => { + const { root, env, state } = fixture(); + const bin = path.join(root, 'portable-bin'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'grep'), `#!/usr/bin/env bun +import { spawnSync } from 'node:child_process'; +const args = process.argv.slice(2); +if (args.some(arg => [...arg.matchAll(/\\{\\d+,(\\d+)\\}/g)].some(match => Number(match[1]) > 255))) { + console.error('REG_BADMAX: repetition exceeds the BSD RE_DUP_MAX of 255'); + process.exit(2); +} +const result = spawnSync(${JSON.stringify(Bun.which('grep'))}, args, { stdio: 'inherit', timeout: 5000 }); +process.exit(result.status ?? 1); +`, { mode: 0o755 }); + const portable = { ...env, PATH: `${bin}${path.delimiter}${env.PATH}` }; + for (const key of ['model_frontier_claude', 'model_smart_claude']) { + for (const value of ['claude-fable-5-1', 'a'.repeat(512)]) { + expect(config(['set', key, value], portable).code).toBe(0); + expect(config(['get', key], portable).stdout).toBe(value); + } + const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8'); + expect(config(['set', key, 'a'.repeat(513)], portable).code).toBe(1); + expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before); + } + }); + + test('defaults, set, resolve again and unset restore', () => { + const { env } = fixture(); + expect(config(['get', 'plan_review_tier'], env)).toMatchObject({ code: 0, stdout: 'frontier' }); + expect(config(['get', 'implementation_tier'], env)).toMatchObject({ code: 0, stdout: 'smart' }); + expect(config(['get', 'model_frontier_claude'], env)).toMatchObject({ code: 0, stdout: '' }); + expect(config(['defaults'], env).stdout).toContain('plan_review_tier:'); + + expect(config(['set', 'plan_review_tier', 'smart'], env).code).toBe(0); + expect(config(['set', 'model_smart_openai', ' gpt-custom '], env).code).toBe(0); + expect(config(['get', 'model_smart_openai'], env).stdout).toBe('gpt-custom'); + expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'smart', requestedModel: 'gpt-custom', source: { kind: 'config' } }); + expect(config(['list'], env).stdout).toContain('model_smart_openai: gpt-custom (set)'); + + expect(config(['unset', 'plan_review_tier'], env).code).toBe(0); + expect(config(['unset', 'model_smart_openai'], env).code).toBe(0); + expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'frontier', requestedModel: 'gpt-6-astra', source: { kind: 'catalog' } }); + }); + + test.each([ + ['plan_review_tier', 'fronteir'], + ['plan_review_tier', ' '], + ['implementation_tier', 'host'], + ['model_frontier_openai', 'gpt 6'], + ['model_frontier_claude', 'a'.repeat(513)], + ])('invalid %s %p is rejected and the old value kept', (key, value) => { + const { env, state } = fixture(); + const good = key.startsWith('model_') ? 'model-1' : 'frontier'; + expect(config(['set', key, good], env).code).toBe(0); + const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8'); + const result = config(['set', key, value], env); + expect(result.code).toBe(1); + expect(result.stderr).toContain('Existing value left unchanged'); + expect(result.stderr).toContain(`gstack-config unset ${key}`); + expect(result.stderr).toContain('#model-policy-config'); + expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before); + }); + + test('set and resolve share one validation rule', () => { + const samples: Array<[PolicyConfigKey, string]> = [ + ['plan_review_tier', 'frontier'], ['plan_review_tier', 'smart'], ['plan_review_tier', 'host'], ['plan_review_tier', 'Smart'], + ['implementation_tier', 'frontier'], ['implementation_tier', 'smart'], ['implementation_tier', 'host'], + ['model_frontier_claude', 'claude-opus-4-6[1m]'], ['model_frontier_claude', 'vendor+claude@2026'], ['model_frontier_claude', 'a'.repeat(512)], + ['model_frontier_claude', 'a'.repeat(513)], ['model_smart_claude', 'claude;x'], ['model_smart_claude', 'claude x'], ['model_smart_claude', '$(id)'], + ['model_frontier_openai', 'gpt-6.1-sol'], ['model_frontier_openai', 'org/model:v1'], ['model_frontier_openai', 'a'.repeat(100)], + ['model_frontier_openai', 'a'.repeat(101)], ['model_smart_openai', 'gpt+6'], ['model_smart_openai', 'gpt@6'], + ]; + const { env } = fixture(); + for (const [key, value] of samples) { + const accepted = config(['set', key, value], env).code === 0; + expect({ key, value: value.slice(0, 20), accepted }).toEqual({ key, value: value.slice(0, 20), accepted: validatePolicyValue(key, value) === null }); + } + }); +}); diff --git a/test/model-policy-freshness.test.ts b/test/model-policy-freshness.test.ts new file mode 100644 index 000000000..ac88d2a02 --- /dev/null +++ b/test/model-policy-freshness.test.ts @@ -0,0 +1,245 @@ +import { describe, expect, test, spyOn } from 'bun:test'; +import { collectFreshnessSources, fetchOfficialSource, FRESHNESS_BOUNDS, FRESHNESS_SOURCES, lifecycleDate, parseOfficialSource, parseOfficialSourceBounded, withinDeadline } from '../lib/model-policy-freshness-sources'; +import { assessFreshness, renderFreshnessReport, storedEvidenceStatus, validEvidence, validLifecycle, validState } from '../lib/model-policy-freshness'; +import { freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts'; + +describe('official recommendation and lifecycle structures', () => { + test('four official fixtures parse, with Mythos and replacement mentions excluded', () => { + const observation = freshnessObservation(); + expect(assessFreshness(observation).status).toBe('current'); + const anthropic = parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models')); + expect(anthropic.recommendations).toEqual({ frontier: 'claude-fable-5-1', smart: 'claude-opus-5-5' }); + expect(anthropic.lineup).not.toContain('claude-mythos-5-1'); + const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle')); + expect(openai.lifecycle?.map(row => row.modelId)).toEqual(['gpt-5.3-codex', 'gpt-old', 'gpt-unicode-date', 'gpt-iso-date', 'gpt-date-floor']); + expect(openai.lifecycle?.find(row => row.modelId === 'gpt-date-floor')).toMatchObject({ deadline: null, supportFloor: '2024-06-13' }); + }); + + test('known-family and actual unknown-family lineup replacements become candidates', () => { + const known = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2').replaceAll('Fable 5.1', 'Fable 5.2'); + const knownReport = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': known })); + expect(knownReport.status).toBe('update-candidate'); + expect(knownReport.candidates[0].reason).toBe('changed-monitored-family-recommendation'); + const unknown = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'mythos-5-1').replaceAll('Fable 5.1', 'Mythos 5.1'); + const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': unknown })); + expect(report.status).toBe('update-candidate'); + expect(report.candidates[0]).toMatchObject({ proposed: 'claude-mythos-5-1', reason: 'unknown-replacement-family-needs-triage' }); + expect(report.complete).toBe(true); + }); + + test('OpenAI full-catalog newer IDs alone are not recommendations', () => { + const original = freshnessFixture('openai-models'); + expect(assessFreshness(freshnessObservation()).candidates).toEqual([]); + const changed = original.replaceAll('gpt-6.1-sol', 'gpt-6.2-sol').replaceAll('GPT-6.1 Sol', 'GPT-6.2 Sol'); + expect(assessFreshness(freshnessObservation('100', undefined, { 'openai-models': changed })).candidates[0]).toMatchObject({ proposed: 'gpt-6.2-sol' }); + }); + + test('active support floors are never retirement deadlines, even after the floor', () => { + const parsed = parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle')); + expect(parsed.lifecycle?.[0]).toMatchObject({ state: 'active', deadline: null, supportFloor: '2027-09-01' }); + expect(assessFreshness(freshnessObservation('100', '2028-10-07T12:00:00.000Z')).state.lifecycle).toEqual([]); + }); + + test('actual API retirement and removal are urgent, incidental partner prose is ignored', () => { + const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + expect(report.status).toBe('update-candidate'); + expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' }); + expect(renderFreshnessReport(report)).toContain('URGENT'); + expect(renderFreshnessReport(report)).toContain('Bedrock/Vertex/custom'); + const removed = assessFreshness(freshnessObservation('100', '2026-11-02T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() })); + expect(removed.state.lifecycle[0].state).toBe('removed'); + }); + + test('OpenAI shutdown rows apply to the deprecated column, not the replacement', () => { + const text = freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`'); + const report = assessFreshness(freshnessObservation('100', undefined, { 'openai-lifecycle': text })); + expect(report.state.lifecycle).toHaveLength(1); + expect(report.state.lifecycle[0]).toMatchObject({ modelId: 'gpt-6-astra', deadline: '2027-04-01' }); + }); + + test('missing required structures and semantic drift fail closed', () => { + expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('Claude API ID', 'Google Cloud ID'))).toThrow('missing-api-id-row'); + expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('for demanding reasoning', 'for unexplained goals'))).toThrow('unrecognized-recommendation'); + expect(() => parseOfficialSource('openai-models', freshnessFixture('openai-models').replace('Balance intelligence and cost.', 'Best new model.'))).toThrow('unrecognized-recommendation'); + expect(() => parseOfficialSource('openai-lifecycle', '# New deprecations\nNo model tables')).toThrow('missing-or-ambiguous-section'); + expect(() => parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', 'gpt-5.3-codex'))).toThrow('unrecognized-deprecated-model-cell'); + expect(() => parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle').replace('Active', 'Probably active'))).toThrow('unrecognized-lifecycle-state'); + expect(() => lifecycleDate('February 30, 2027')).toThrow('malformed-lifecycle-date'); + expect(() => lifecycleDate('tomorrow')).toThrow('malformed-lifecycle-date'); + expect(() => parseOfficialSource('anthropic-models', `${freshnessFixture('anthropic-models')}\n## Compare models\nDuplicate`)).toThrow('ambiguous-section'); + expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('## Using the Models API', 'Use [Other](https://platform.claude.com/docs/en/models/opus-5-5/overview) for demanding reasoning.\n\n## Using the Models API'))).toThrow('unrecognized-recommendation'); + }); +}); + +describe('bounded fixed-source fetching', () => { + test('exact byte limit passes and a streamed byte beyond it fails without trusting content-length', async () => { + const atCap = 'x'.repeat(FRESHNESS_BOUNDS.responseBytes); + expect((await fetchOfficialSource('openai-models', async () => new Response(atCap))).bytes).toBe(2_097_152); + await expect(fetchOfficialSource('openai-models', async () => new Response(`${atCap}x`))).rejects.toThrow('response-too-large'); + await expect(fetchOfficialSource('openai-models', async () => new Response('x', { headers: { 'content-length': '2097153' } }))).rejects.toThrow('response-too-large'); + expect(() => parseOfficialSource('openai-models', `${atCap}x`)).toThrow('response-too-large'); + }); + + test('all redirect destinations are checked; three redirects pass, four fail', async () => { + const credentialUrl = new URL(FRESHNESS_SOURCES[2].url); + credentialUrl.username = 'user'; + credentialUrl.password = 'pass'; + for (const location of ['https://evil.example/models.md', 'http://developers.openai.com/api/docs/models.md', credentialUrl.href, 'https://developers.openai.com/api/docs/models.md?untrusted=1']) { + let calls = 0; + await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location } }); })).rejects.toThrow('redirect-not-allowlisted'); + expect(calls).toBe(1); + } + let calls = 0; + expect((await fetchOfficialSource('openai-models', async () => ++calls <= 3 ? new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }) : new Response('data'))).text).toBe('data'); + calls = 0; + await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }); })).rejects.toThrow('redirect-limit'); + expect(calls).toBe(4); + }); + + test('per-response and whole-check clocks are installed at the exact approved bounds', async () => { + const installed: number[] = []; + const real = globalThis.setTimeout; + const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, milliseconds: number) => { + installed.push(milliseconds); + return real(callback, milliseconds === 30_000 ? 5 : milliseconds); + }) as typeof setTimeout); + try { + const result = await collectFreshnessSources(async () => new Promise(() => {})); + expect(result.every(source => source.error === 'deadline-exceeded')).toBe(true); + expect(installed.filter(ms => ms === 30_000)).toHaveLength(4); + expect(installed).toContain(180_000); + } finally { timers.mockRestore(); } + const controller = new AbortController(); + await expect(withinDeadline(new Promise(() => {}), 5, controller)).rejects.toThrow('deadline-exceeded'); + expect(controller.signal.aborted).toBe(true); + }); + + test('only four fixed sources are fetched and failures never become parseable success', async () => { + const urls: string[] = []; + const result = await collectFreshnessSources(async (url, init) => { + urls.push(url); + expect(init.redirect).toBe('manual'); + const id = FRESHNESS_SOURCES.find(source => source.url === url)!.id; + return id === 'openai-lifecycle' ? new Response('authentication needed', { status: 403 }) : new Response(freshnessFixture(id)); + }); + expect(urls.sort()).toEqual(FRESHNESS_SOURCES.map(source => source.url).sort()); + expect(result.find(source => source.id === 'openai-lifecycle')?.error).toBe('http-403'); + const observation = freshnessObservation(); + observation.sources = result; + expect(assessFreshness(observation).status).toBe('unknown/source-unavailable'); + }); + + test('CPU-bound parsing is terminable and the complete-check deadline stops all blocked parsers', async () => { + let terminations = 0; + const createWorker = () => ({ postMessage() {}, terminate() { terminations++; }, onmessage: null, onerror: null }) as unknown as Worker; + const controller = new AbortController(); + const blocked = parseOfficialSourceBounded('openai-models', freshnessFixture('openai-models'), controller.signal, createWorker); + controller.abort(); + await expect(blocked).rejects.toThrow('deadline-exceeded'); + expect(terminations).toBe(1); + const real = globalThis.setTimeout; + const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => real(callback, ms === 180_000 ? 5 : ms)) as typeof setTimeout); + try { + const result = await collectFreshnessSources(async url => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id)), createWorker); + expect(result.every(source => source.error && !source.receipt)).toBe(true); + } finally { timers.mockRestore(); } + expect(terminations).toBe(5); + }); +}); + +describe('trustworthy evidence and deterministic statuses', () => { + test('source-check evidence is separate from catalog qualification and survives artifact expiry', () => { + const report = assessFreshness(freshnessObservation()); + expect(validState(report.state)).toBe(true); + expect(validEvidence(report.state.lastSuccess)).toBe(true); + expect(report.state.lastSuccess?.nextCheckDueBy).toBe('2026-10-14T12:00:00.000Z'); + expect(report.state.lastSuccess?.sources).toHaveLength(4); + expect(report.state.lastSuccess?.catalog).toEqual(freshnessObservation().catalog); + expect(storedEvidenceStatus(freshnessObservation().catalog, JSON.parse(JSON.stringify(report.state.lastSuccess)), '2026-10-08T12:00:00.000Z')).toBe('current'); + expect(renderFreshnessReport(report)).toContain('not automatic liveness monitoring'); + }); + + test('missing/corrupt/mismatched/future evidence is unknown; thirty-day staleness has an exact boundary', () => { + const observation = freshnessObservation(); + const evidence = assessFreshness(observation).state.lastSuccess!; + expect(storedEvidenceStatus(observation.catalog, null, observation.checkedAt)).toBe('unknown/source-unavailable'); + expect(storedEvidenceStatus({ ...observation.catalog, sha256: 'b'.repeat(64) }, evidence, observation.checkedAt)).toBe('unknown/source-unavailable'); + expect(storedEvidenceStatus(observation.catalog, { ...evidence, parserVersion: 99 }, observation.checkedAt)).toBe('unknown/source-unavailable'); + expect(storedEvidenceStatus(observation.catalog, evidence, '2026-10-06T12:00:00.000Z')).toBe('unknown/source-unavailable'); + expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.000Z')).toBe('current'); + expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.001Z')).toBe('stale'); + }); + + test('partial recovery cannot refresh evidence or erase an unresolved retirement', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const next = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'openai-lifecycle': null }), initial.state); + expect(next.status).toBe('unknown/source-unavailable'); + expect(next.state.lastSuccess).toEqual(initial.state.lastSuccess); + expect(next.state.lifecycle).toEqual(initial.state.lifecycle); + const old = assessFreshness(freshnessObservation('102', '2026-11-08T12:00:00.000Z', { 'openai-lifecycle': null }), next.state); + expect(old.evidenceStatus).toBe('stale'); + expect(old.status).toBe('unknown/source-unavailable'); + const recovered = assessFreshness(freshnessObservation('103', '2026-11-09T12:00:00.000Z'), old.state); + expect(recovered.status).toBe('update-candidate'); + expect(recovered.state.lifecycle).toEqual(old.state.lifecycle); + }); + + test('corrupt duplicate lifecycle keys and source receipts cannot certify current', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + expect(validLifecycle([initial.state.lifecycle[0], initial.state.lifecycle[0]])).toBe(false); + const observation = freshnessObservation(); + observation.sources[0].receipt!.finalUrl = 'https://evil.example/docs'; + expect(assessFreshness(observation).status).toBe('unknown/source-unavailable'); + const evidence = structuredClone(initial.state.lastSuccess!); + evidence.sources[0] = evidence.sources[1]; + expect(validEvidence(evidence)).toBe(false); + expect(validState({ ...initial.state, lastSuccess: evidence })).toBe(false); + }); + + test('a partial run cannot downgrade an unresolved retirement to legacy or extend its deadline', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const legacy = freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active', 'claude-fable-5-1 | Legacy'); + const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': legacy, 'openai-lifecycle': null }), initial.state); + expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' }); + expect(report.state.lastSuccess).toEqual(initial.state.lastSuccess); + }); + + test('only a complete check of a reviewed replacement catalog resolves removed identities', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const replacement = freshnessObservation('101', '2026-10-08T12:00:00.000Z'); + replacement.catalog.models[0].modelId = 'claude-fable-5-2'; + replacement.catalog.sha256 = 'b'.repeat(64); + replacement.sources = freshnessObservation('101', replacement.checkedAt, { + 'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'), + 'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'), + }).sources; + const incomplete = structuredClone(replacement); + incomplete.sources[3] = { id: 'openai-lifecycle', error: 'http-503' }; + expect(assessFreshness(incomplete, initial.state).state.lifecycle[0].resolvedByCatalog).toBeUndefined(); + const complete = assessFreshness(replacement, initial.state); + expect(complete.status).toBe('current'); + expect(complete.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64)); + }); + + test('complete catalog rollback reopens the historical retirement even when the current source row is active', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const replacement = assessFreshness(freshnessReplacementObservation(), initial.state); + expect(replacement.status).toBe('current'); + expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64)); + const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), replacement.state); + expect(rollback.status).toBe('update-candidate'); + expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01', firstObservedAt: initial.state.lifecycle[0].firstObservedAt }); + expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined(); + expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64)); + }); + + test('source-unavailable catalog rollback reopens history without refreshing prior replacement evidence', () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const replacement = assessFreshness(freshnessReplacementObservation(), initial.state); + const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), replacement.state); + expect(rollback.status).toBe('unknown/source-unavailable'); + expect(rollback.state.lastSuccess).toEqual(replacement.state.lastSuccess); + expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' }); + expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined(); + }); +}); diff --git a/test/model-policy-workflow.test.ts b/test/model-policy-workflow.test.ts new file mode 100644 index 000000000..5a0bd0670 --- /dev/null +++ b/test/model-policy-workflow.test.ts @@ -0,0 +1,547 @@ +import { describe, expect, test, spyOn } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { MODEL_CATALOG, modelCatalogSha256 } from '../lib/model-catalog'; +import { assessFreshness, freshnessCatalogIdentity, validLifecycle, validState } from '../lib/model-policy-freshness'; +import { FRESHNESS_SOURCES, FRESHNESS_BOUNDS, sha256 } from '../lib/model-policy-freshness-sources'; +import { FRESHNESS_ISSUE_AUTHOR, FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, publishFreshness, readFreshnessIssue, writeFreshnessIssue, type FreshnessGitHub, type TrackingIssue } from '../lib/model-policy-freshness-publication'; +import { freshnessGitHub } from '../lib/model-policy-freshness-github'; +import { runModelPolicyFreshness } from '../scripts/model-policy-freshness'; +import { freshnessCatalog, freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts'; + +function mockGitHub(initial: (Omit & { author?: string })[] = []) { + const issues: TrackingIssue[] = structuredClone(initial).map(issue => ({ author: FRESHNESS_ISSUE_AUTHOR, ...issue })); + const calls: string[] = []; + let identity = { defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 }; + const github: FreshnessGitHub = { + async defaultCatalog() { calls.push('catalog'); return identity; }, + async listIssues() { calls.push('list'); return structuredClone(issues); }, + async readIssue(number) { calls.push('read'); return structuredClone(issues.find(issue => issue.number === number)!); }, + async createIssue(body) { calls.push('create'); const issue: TrackingIssue = { number: issues.length + 1, body, state: 'open', author: FRESHNESS_ISSUE_AUTHOR }; issues.push(issue); return structuredClone(issue); }, + async updateIssue(number, update) { calls.push('update'); Object.assign(issues.find(issue => issue.number === number)!, update); }, + }; + return { github, issues, calls, setIdentity(value: typeof identity) { identity = value; } }; +} + +describe('stable-marker issue ownership and evidence lifecycle', () => { + test('healthy bootstrap creates one closed issue, retains evidence when closed, and repeat publication is idempotent', async () => { + const api = mockGitHub(); + const observation = freshnessObservation(); + const created = await publishFreshness(observation, api.github, 'refs/heads/main'); + expect(created.action).toBe('created'); + expect(api.issues).toHaveLength(1); + expect(api.issues[0].state).toBe('closed'); + expect(api.issues[0].body).toContain(FRESHNESS_ISSUE_MARKER); + expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess?.checkedAt).toBe(observation.checkedAt); + const body = api.issues[0].body; + expect((await publishFreshness(observation, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation'); + expect(api.issues[0].body).toBe(body); + const next = freshnessObservation('101', '2026-10-08T12:00:00.000Z'); + expect((await publishFreshness(next, api.github, 'refs/heads/main')).action).toBe('updated'); + expect(api.issues).toHaveLength(1); + expect(api.issues[0].state).toBe('closed'); + }); + + test('marker owns the issue, title collisions and human body/disposition text are untouched', async () => { + const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation())); + const humanPrefix = 'Human notes: retain this default until qualification.\n'; + const humanSuffix = '\nHuman disposition: mitigation pending; do not delete.\n'; + const api = mockGitHub([ + { number: 40, body: 'Model-policy freshness: advisory maintenance', state: 'open' }, + { number: 41, body: humanPrefix + initial + humanSuffix, state: 'closed' }, + ]); + await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); + expect(api.issues[0].body).toBe('Model-policy freshness: advisory maintenance'); + expect(api.issues[1].body.startsWith(humanPrefix)).toBe(true); + expect(api.issues[1].body.endsWith(humanSuffix)).toBe(true); + expect(api.issues[1].state).toBe('open'); + expect(readFreshnessIssue(api.issues[1].body).lifecycle[0].state).toBe('deprecated'); + }); + + test('retirement survives source failure and partial recovery; issue never closes on either', async () => { + const api = mockGitHub(); + await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); + const evidence = readFreshnessIssue(api.issues[0].body).state!.lastSuccess; + const failure = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); + expect(failure.report?.status).toBe('unknown/source-unavailable'); + expect(api.issues[0].state).toBe('open'); + expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess).toEqual(evidence); + await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'openai-lifecycle': null }), api.github, 'refs/heads/main'); + expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].deadline).toBe('2026-11-01'); + expect(api.issues[0].state).toBe('open'); + }); + + test('both complete and source-unavailable catalog rollbacks reopen the closed replacement issue', async () => { + for (const unavailable of [false, true]) { + const api = mockGitHub(); + await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); + const replacement = freshnessReplacementObservation(); + api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); + await publishFreshness(replacement, api.github, 'refs/heads/main'); + expect(api.issues[0].state).toBe('closed'); + expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBe(replacement.catalog.sha256); + api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 }); + const rollback = freshnessObservation('102', '2026-10-09T12:00:00.000Z', unavailable ? { 'anthropic-lifecycle': null, 'openai-lifecycle': null } : {}); + const result = await publishFreshness(rollback, api.github, 'refs/heads/main'); + expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'update-candidate'); + expect(api.issues[0].state).toBe('open'); + expect(api.issues).toHaveLength(1); + const stored = readFreshnessIssue(api.issues[0].body); + expect(stored.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' }); + expect(stored.lifecycle[0].resolvedByCatalog).toBeUndefined(); + if (unavailable) expect(stored.state?.lastSuccess?.catalog.sha256).toBe(replacement.catalog.sha256); + } + }); + + test('corrupt state salvages independent lifecycle history and requires explicit manual repair', async () => { + const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }))); + const broken = initial.replace(/(\n)[^\n]+/, '$1{broken'); + const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]); + expect(readFreshnessIssue(api.issues[0].body).state).toBeNull(); + expect(readFreshnessIssue(api.issues[0].body).lifecycle).toHaveLength(1); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main'); + expect(result.report?.status).toBe('unknown/source-unavailable'); + expect(result.report?.state.lastSuccess).toBeNull(); + expect(result.report?.state.lifecycle).toHaveLength(1); + expect(api.issues[0].state).toBe('open'); + expect(api.issues[0].body).toContain('Human retain decision'); + expect(api.issues[0].body).toContain('Human footer'); + expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64')); + const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main'); + expect(next.report?.state.recoveryRequired).toBe(true); + }); + + test('valid empty state cannot erase independent retirement history during partial publication', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const state = { ...initial.state, lifecycle: [] }; + const broken = writeFreshnessIssue('', initial).replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)); + const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]); + const recovered = readFreshnessIssue(api.issues[0].body); + expect(validState(recovered.state)).toBe(true); + expect(recovered.state?.lifecycle).toEqual([]); + expect(recovered.lifecycle).toEqual(initial.state.lifecycle); + expect(recovered.recoveryRequired).toBe(true); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); + expect(result.action).toBe('updated'); + expect(result.report?.state.lifecycle).toEqual(initial.state.lifecycle); + expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); + expect(result.report?.status).toBe('unknown/source-unavailable'); + expect(result.report?.complete).toBe(false); + const stored = readFreshnessIssue(api.issues[0].body); + expect(stored.state?.lifecycle).toEqual(initial.state.lifecycle); + expect(stored.lifecycle).toEqual(initial.state.lifecycle); + expect(stored.recoveryRequired).toBe(true); + expect(api.issues[0].state).toBe('open'); + expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); + expect(api.issues[0].body.startsWith('Human retain decision\n')).toBe(true); + expect(api.issues[0].body.endsWith('\nHuman footer')).toBe(true); + expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64')); + const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main'); + expect(next.report?.state.lifecycle).toEqual(initial.state.lifecycle); + expect(next.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); + expect(next.report?.state.recoveryRequired).toBe(true); + expect(next.report?.complete).toBe(false); + expect(api.issues[0].state).toBe('open'); + }); + + test('distinct valid state and history findings are both retained regardless of copy ownership', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { + 'anthropic-lifecycle': retirementFixture(), + 'openai-lifecycle': freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`'), + })); + expect(initial.state.lifecycle).toHaveLength(2); + for (const reversed of [false, true]) { + const findings = reversed ? initial.state.lifecycle.toReversed() : initial.state.lifecycle; + const state = { ...initial.state, lifecycle: [findings[0]] }; + const history = [findings[1]]; + const body = writeFreshnessIssue('', initial) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); + const recovered = readFreshnessIssue(body); + expect(validState(recovered.state)).toBe(true); + expect(validLifecycle(recovered.lifecycle)).toBe(true); + expect(recovered.state?.lifecycle).toEqual(state.lifecycle); + expect(recovered.lifecycle).toEqual(history); + const api = mockGitHub([{ number: 7, body, state: 'open' }]); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); + expect(result.report?.state.lifecycle).toEqual(initial.state.lifecycle); + expect(readFreshnessIssue(api.issues[0].body).lifecycle).toEqual(initial.state.lifecycle); + expect(result.report?.state.recoveryRequired).toBe(true); + expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); + expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); + expect(api.issues[0].body).toContain('- URGENT: gpt-6-astra is deprecated; removal deadline 2027-04-01.'); + expect(api.issues[0].state).toBe('open'); + } + }); + + test('conflicting copies retain the strongest severity, earliest deadline and full observation chronology', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const finding = initial.state.lifecycle[0]; + const severe = { ...finding, state: 'removed' as const, deadline: '2026-10-01', firstObservedAt: '2026-09-15T12:00:00.000Z', lastObservedAt: '2026-10-05T12:00:00.000Z' }; + const early = { ...finding, state: 'legacy' as const, deadline: '2026-09-30', firstObservedAt: '2026-09-01T12:00:00.000Z', lastObservedAt: '2026-10-06T12:00:00.000Z', resolvedByCatalog: 'b'.repeat(64) }; + for (const reversed of [false, true]) { + const state = { ...initial.state, lifecycle: [reversed ? early : severe] }; + const history = [reversed ? severe : early]; + const body = writeFreshnessIssue('', initial) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); + expect(validState(readFreshnessIssue(body).state)).toBe(true); + expect(validLifecycle(readFreshnessIssue(body).lifecycle)).toBe(true); + const api = mockGitHub([{ number: 7, body, state: 'closed' }]); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null }), api.github, 'refs/heads/main'); + const expected = [{ ...severe, deadline: early.deadline, firstObservedAt: early.firstObservedAt, lastObservedAt: early.lastObservedAt }]; + expect(result.report?.state.lifecycle).toEqual(expected); + expect(validState(result.report?.state)).toBe(true); + expect(readFreshnessIssue(api.issues[0].body).lifecycle).toEqual(expected); + expect(result.report?.state.recoveryRequired).toBe(true); + expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); + expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is removed; removal deadline 2026-09-30.'); + expect(api.issues[0].state).toBe('open'); + } + }); + + test('conflicting historical dispositions stay unresolved until manual repair even after a complete replacement check', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const replacement = freshnessReplacementObservation(); + const replaced = assessFreshness(replacement, initial.state); + const finding = replaced.state.lifecycle[0]; + expect(finding.resolvedByCatalog).toBe(replacement.catalog.sha256); + for (const disposition of [undefined, 'd'.repeat(64)]) for (const reversed of [false, true]) for (const unavailable of [false, true]) { + const disagreement = { ...finding, resolvedByCatalog: disposition }; + const state = { ...replaced.state, lifecycle: [reversed ? disagreement : finding] }; + const history = [reversed ? finding : disagreement]; + const body = writeFreshnessIssue('', replaced) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) + .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); + const api = mockGitHub([{ number: 7, body, state: 'closed' }]); + api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); + const next = structuredClone(replacement); + next.run.id = '102'; + next.checkedAt = '2026-10-09T12:00:00.000Z'; + if (unavailable) next.sources[3] = { id: 'openai-lifecycle', error: 'http-503' }; + const result = await publishFreshness(next, api.github, 'refs/heads/main'); + expect(result.report?.state.lifecycle).toHaveLength(1); + expect(result.report?.state.lifecycle[0].resolvedByCatalog).toBeUndefined(); + expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBeUndefined(); + expect(result.report?.state.recoveryRequired).toBe(true); + expect(result.report?.state.lastSuccess).toEqual(replaced.state.lastSuccess); + expect(result.report?.status).toBe('unknown/source-unavailable'); + expect(result.report?.complete).toBe(false); + expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); + expect(api.issues[0].state).toBe('open'); + } + }); + + test('matching resolved history remains resolved without recovery and only complete evidence closes the issue', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + const replacement = freshnessReplacementObservation(); + const replaced = assessFreshness(replacement, initial.state); + for (const unavailable of [false, true]) { + const api = mockGitHub([{ number: 7, body: writeFreshnessIssue('', replaced), state: 'closed' }]); + api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); + const next = structuredClone(replacement); + next.run.id = '102'; + next.checkedAt = '2026-10-09T12:00:00.000Z'; + if (unavailable) next.sources[3] = { id: 'openai-lifecycle', error: 'http-503' }; + const result = await publishFreshness(next, api.github, 'refs/heads/main'); + expect(result.report?.state.lifecycle).toEqual(replaced.state.lifecycle); + expect(result.report?.state.recoveryRequired).toBe(false); + expect(result.report?.complete).toBe(!unavailable); + expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'current'); + expect(api.issues[0].state).toBe(unavailable ? 'open' : 'closed'); + expect(api.issues[0].body).not.toContain('- URGENT:'); + } + }); + + test('missing machine state is recovery, malformed owned boundary is not overwritten, and output is bounded', async () => { + const api = mockGitHub([{ number: 9, body: `${FRESHNESS_ISSUE_MARKER}\nMaintainer notes`, state: 'closed' }]); + const result = await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main'); + expect(result.report?.status).toBe('unknown/source-unavailable'); + expect(api.issues[0].state).toBe('open'); + expect(api.issues[0].body).toContain('Maintainer notes'); + expect(() => writeFreshnessIssue(`${FRESHNESS_ISSUE_MARKER}\n${FRESHNESS_REGION_BEGIN}\nbroken`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region'); + expect(() => writeFreshnessIssue('x'.repeat(FRESHNESS_BOUNDS.issueBytes), assessFreshness(freshnessObservation()))).toThrow('tracking-issue-size-limit'); + expect(() => writeFreshnessIssue(`${FRESHNESS_REGION_END}\n${FRESHNESS_REGION_BEGIN}`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region'); + }); + + test('duplicate markers fail visibly rather than selecting an arbitrary issue', async () => { + const api = mockGitHub([{ number: 1, body: FRESHNESS_ISSUE_MARKER, state: 'open' }, { number: 2, body: FRESHNESS_ISSUE_MARKER, state: 'closed' }]); + await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('duplicate-tracking-marker'); + expect(api.calls).not.toContain('update'); + expect(api.calls).not.toContain('create'); + }); + + test('user-created marker copies never become authority or trigger replacement bootstrap', async () => { + const body = writeFreshnessIssue('', assessFreshness(freshnessObservation())); + for (const includeOwned of [false, true]) { + const api = mockGitHub([{ number: 1, body, state: 'closed', author: 'untrusted-user' }, ...(includeOwned ? [{ number: 2, body, state: 'closed' as const }] : [])]); + await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('untrusted-tracking-marker-owner'); + expect(api.calls).not.toContain('read'); + expect(api.calls).not.toContain('update'); + expect(api.calls).not.toContain('create'); + } + }); + + test('CRLF and CR-only marker lines retain retirement history and publish to the same issue on source failure', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + for (const newline of ['\r\n', '\r']) { + const prefix = `Human prefix remains byte-identical.${newline}${newline}`; + const suffix = `${newline}${newline}Human suffix remains byte-identical.${newline}`; + const body = prefix + writeFreshnessIssue('', initial).replaceAll('\n', newline) + suffix; + const api = mockGitHub([{ number: 17, body, state: 'open' }]); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); + expect(result.action).toBe('updated'); + expect(result.issueNumber).toBe(17); + expect(api.issues).toHaveLength(1); + expect(api.calls).not.toContain('create'); + expect(api.issues[0].body.startsWith(prefix)).toBe(true); + expect(api.issues[0].body.endsWith(suffix)).toBe(true); + const stored = readFreshnessIssue(api.issues[0].body); + expect(stored.lifecycle).toEqual(initial.state.lifecycle); + expect(stored.state?.lastSuccess).toEqual(initial.state.lastSuccess); + expect(result.report?.status).toBe('unknown/source-unavailable'); + } + }); + + test('inline and quoted marker mentions preserve human prefix and suffix outside the real region', async () => { + const initial = assessFreshness(freshnessObservation()); + const prefix = `Human note mentioning ${FRESHNESS_REGION_BEGIN} inline; preserve this entire sentence.\r\n> ${FRESHNESS_REGION_END}\r\nQuoted '${FRESHNESS_REGION_BEGIN}' and '${FRESHNESS_REGION_END}' are human prose.\r\n\r\n`; + const suffix = `\r\nHuman suffix mentions ${FRESHNESS_REGION_END} inline.\r\n> ${FRESHNESS_REGION_BEGIN}\r\n`; + const body = prefix + writeFreshnessIssue('', initial) + suffix; + const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), initial.state); + const rewritten = writeFreshnessIssue(body, report); + expect(rewritten.startsWith(prefix)).toBe(true); + expect(rewritten.endsWith(suffix)).toBe(true); + expect(readFreshnessIssue(rewritten).state).toEqual(report.state); + const api = mockGitHub([{ number: 19, body, state: 'closed' }]); + const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main'); + expect(result.action).toBe('updated'); + expect(api.issues).toHaveLength(1); + expect(api.issues[0].body.startsWith(prefix)).toBe(true); + expect(api.issues[0].body.endsWith(suffix)).toBe(true); + }); + + test('malformed or missing top markers with owned evidence never become a healthy bootstrap', async () => { + const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); + for (const marker of ['', `${FRESHNESS_ISSUE_MARKER} `, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace('