From a1da77e6d45320f5066fd2438d928fc94b36ab73 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 07:04:56 -0800 Subject: [PATCH 1/5] v1.94.20.0 test(review): one file per review and harness pin, so independent repins merge cleanly (GST-1084) (#3903) * test(review): one file per review and harness pin, so independent repins merge cleanly (GST-1084) The reviewer-brief pins move from the table in test/review-packets-removed.test.ts to test/fixtures/review-pins/{files,replays,brief-bodies}/*.json, and the harness pins from scripts/harness-version.json to scripts/harness-pins/.json. Every entry is ported unchanged. New tests prove a stale pin still fails and that two repins of different pins merge cleanly with a real git merge-tree. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (GST-1084) - unadmitted() and the specialist coverage use Object.hasOwn, so an inherited name such as toString is not an admitted replay - MUST_PIN names every non-specialist reviewer source; deleting one of those pins now fails - writeManifest writes the pins before the manifest head; pin parse errors name the file - mergePinEdits removes its repository when setup throws, and the merge tests clean up every merge they made Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: name the changelog fragment for the branch (GST-1084) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: the harness guard pins one file per harness file (GST-1084) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-release): GST-1031 re-pin route reads per-file harness pins (GST-1084) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * changelog: GST-1031 route reads per-file harness pins Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(test-audit): harness guard pins 42 files Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(harness-version): a pin ahead of the manifest version counts as drift (GST-1084) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 10 +- docs/test-audit-2026-10.md | 8 +- lib/ship-release.ts | 20 +- package.json | 2 +- scripts/bump-harness-version.ts | 58 +++-- .../scripts/lib/paid-cases.ts.json | 1 + .../scripts/lib/paid-plan.ts.json | 1 + .../scripts/lib/paid-report.ts.json | 1 + .../scripts/lib/published-text.ts.json | 1 + .../scripts/lib/shard-engine.ts.json | 1 + .../scripts/test-paid-shards.ts.json | 1 + .../scripts/test-pr-profile.ts.json | 1 + .../scripts/test-strict-output.ts.json | 1 + .../test/helpers/agent-sdk-runner.ts.json | 1 + .../test/helpers/auq-substance-panel.ts.json | 1 + .../test/helpers/claude-pty-runner.ts.json | 1 + .../test/helpers/codex-host-runner.ts.json | 1 + .../test/helpers/codex-session-runner.ts.json | 1 + .../test/helpers/e2e-gate.ts.json | 1 + .../test/helpers/e2e-helpers.ts.json | 1 + .../test/helpers/eval-budgets.ts.json | 1 + .../test/helpers/eval-store.ts.json | 1 + .../test/helpers/hermetic-env.ts.json | 1 + .../test/helpers/llm-judge.ts.json | 1 + .../test/helpers/paid-test-set.ts.json | 1 + .../test/helpers/pty-screen.ts.json | 1 + .../harness-pins/test/helpers/pty/auq.ts.json | 1 + .../test/helpers/pty/binary.ts.json | 1 + .../test/helpers/pty/boundaries.ts.json | 1 + .../test/helpers/pty/classify.ts.json | 1 + .../test/helpers/pty/fake-session.ts.json | 1 + .../test/helpers/pty/judge.ts.json | 1 + .../test/helpers/pty/launch.ts.json | 1 + .../test/helpers/pty/plan-native.ts.json | 1 + .../test/helpers/pty/runners/counting.ts.json | 1 + .../test/helpers/pty/runners/floor.ts.json | 1 + .../helpers/pty/runners/observation.ts.json | 1 + .../test/helpers/pty/screen.ts.json | 1 + .../test/helpers/pty/session.ts.json | 1 + .../test/helpers/pty/transcript-cost.ts.json | 1 + .../test/helpers/session-drain-policy.ts.json | 1 + .../test/helpers/session-ledger.ts.json | 1 + .../test/helpers/session-runner.ts.json | 1 + .../test/helpers/skill-fixture.ts.json | 1 + .../test/helpers/skill-startup.ts.json | 1 + .../test/helpers/test-selection.ts.json | 1 + .../test/helpers/touchfiles.ts.json | 1 + scripts/harness-version.json | 46 +--- .../review-pins/brief-bodies/fixtures.json | 1 + .../review-pins/brief-bodies/framing.json | 1 + .../review-pins/brief-bodies/native.json | 1 + .../review-pins/brief-bodies/outside.json | 1 + .../review-pins/brief-bodies/redTeam.json | 1 + .../review-pins/brief-bodies/specialist.json | 1 + .../files/lib/review-depth.ts.json | 1 + .../review-pins/files/lib/review-risk.ts.json | 1 + .../files/review/checklist.md.json | 1 + .../files/review/design-checklist.md.json | 1 + .../files/review/gate-integrity.md.json | 1 + .../files/review/sections/adversarial.md.json | 1 + .../files/review/sections/review-army.md.json | 1 + .../review/specialists/api-contract.md.json | 1 + .../review/specialists/data-migration.md.json | 1 + .../specialists/maintainability.md.json | 1 + .../review/specialists/performance.md.json | 1 + .../files/review/specialists/red-team.md.json | 1 + .../files/review/specialists/security.md.json | 1 + .../review/specialists/simplification.md.json | 1 + .../review/specialists/state-machines.md.json | 1 + .../files/review/specialists/targeted.md.json | 1 + .../files/review/specialists/testing.md.json | 1 + .../files/scripts/resolvers/fix-check.ts.json | 1 + .../files/ship/sections/adversarial.md.json | 1 + .../files/ship/sections/review-army.md.json | 1 + test/fixtures/review-pins/replays/3638.json | 1 + test/fixtures/review-pins/replays/G1.json | 1 + .../fixtures/review-pins/replays/GST-257.json | 1 + .../fixtures/review-pins/replays/GST-464.json | 1 + .../replays/GST-508+GST-464+GST-967.json | 1 + .../review-pins/replays/GST-508+GST-464.json | 1 + .../replays/GST-508+R51+GST-911+GST-967.json | 1 + .../replays/GST-508+R51+GST-911.json | 1 + .../review-pins/replays/GST-508+R51.json | 1 + .../fixtures/review-pins/replays/GST-508.json | 1 + .../fixtures/review-pins/replays/GST-850.json | 1 + .../fixtures/review-pins/replays/GST-898.json | 1 + .../review-pins/replays/GST-911-redos.json | 1 + .../fixtures/review-pins/replays/GST-911.json | 1 + .../fixtures/review-pins/replays/GST-940.json | 1 + .../fixtures/review-pins/replays/GST-942.json | 1 + .../fixtures/review-pins/replays/GST-967.json | 1 + .../fixtures/review-pins/replays/GST-986.json | 1 + .../review-pins/replays/GSTA-224-H1.json | 1 + test/fixtures/review-pins/replays/L.json | 1 + test/fixtures/review-pins/replays/PR-a.json | 1 + test/fixtures/review-pins/replays/R51.json | 1 + test/harness-version.test.ts | 109 +++++++- test/helpers/free-fixtures-data.ts | 1 + test/helpers/pin-merge-fixture.ts | 41 +++ test/review-depth.test.ts | 2 +- test/review-packets-removed.test.ts | 233 ++++++++++++------ test/ship-release.test.ts | 23 +- 105 files changed, 494 insertions(+), 165 deletions(-) create mode 100644 scripts/harness-pins/scripts/lib/paid-cases.ts.json create mode 100644 scripts/harness-pins/scripts/lib/paid-plan.ts.json create mode 100644 scripts/harness-pins/scripts/lib/paid-report.ts.json create mode 100644 scripts/harness-pins/scripts/lib/published-text.ts.json create mode 100644 scripts/harness-pins/scripts/lib/shard-engine.ts.json create mode 100644 scripts/harness-pins/scripts/test-paid-shards.ts.json create mode 100644 scripts/harness-pins/scripts/test-pr-profile.ts.json create mode 100644 scripts/harness-pins/scripts/test-strict-output.ts.json create mode 100644 scripts/harness-pins/test/helpers/agent-sdk-runner.ts.json create mode 100644 scripts/harness-pins/test/helpers/auq-substance-panel.ts.json create mode 100644 scripts/harness-pins/test/helpers/claude-pty-runner.ts.json create mode 100644 scripts/harness-pins/test/helpers/codex-host-runner.ts.json create mode 100644 scripts/harness-pins/test/helpers/codex-session-runner.ts.json create mode 100644 scripts/harness-pins/test/helpers/e2e-gate.ts.json create mode 100644 scripts/harness-pins/test/helpers/e2e-helpers.ts.json create mode 100644 scripts/harness-pins/test/helpers/eval-budgets.ts.json create mode 100644 scripts/harness-pins/test/helpers/eval-store.ts.json create mode 100644 scripts/harness-pins/test/helpers/hermetic-env.ts.json create mode 100644 scripts/harness-pins/test/helpers/llm-judge.ts.json create mode 100644 scripts/harness-pins/test/helpers/paid-test-set.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty-screen.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/auq.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/binary.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/boundaries.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/classify.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/fake-session.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/judge.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/launch.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/plan-native.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/runners/counting.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/runners/floor.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/runners/observation.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/screen.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/session.ts.json create mode 100644 scripts/harness-pins/test/helpers/pty/transcript-cost.ts.json create mode 100644 scripts/harness-pins/test/helpers/session-drain-policy.ts.json create mode 100644 scripts/harness-pins/test/helpers/session-ledger.ts.json create mode 100644 scripts/harness-pins/test/helpers/session-runner.ts.json create mode 100644 scripts/harness-pins/test/helpers/skill-fixture.ts.json create mode 100644 scripts/harness-pins/test/helpers/skill-startup.ts.json create mode 100644 scripts/harness-pins/test/helpers/test-selection.ts.json create mode 100644 scripts/harness-pins/test/helpers/touchfiles.ts.json create mode 100644 test/fixtures/review-pins/brief-bodies/fixtures.json create mode 100644 test/fixtures/review-pins/brief-bodies/framing.json create mode 100644 test/fixtures/review-pins/brief-bodies/native.json create mode 100644 test/fixtures/review-pins/brief-bodies/outside.json create mode 100644 test/fixtures/review-pins/brief-bodies/redTeam.json create mode 100644 test/fixtures/review-pins/brief-bodies/specialist.json create mode 100644 test/fixtures/review-pins/files/lib/review-depth.ts.json create mode 100644 test/fixtures/review-pins/files/lib/review-risk.ts.json create mode 100644 test/fixtures/review-pins/files/review/checklist.md.json create mode 100644 test/fixtures/review-pins/files/review/design-checklist.md.json create mode 100644 test/fixtures/review-pins/files/review/gate-integrity.md.json create mode 100644 test/fixtures/review-pins/files/review/sections/adversarial.md.json create mode 100644 test/fixtures/review-pins/files/review/sections/review-army.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/api-contract.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/data-migration.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/maintainability.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/performance.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/red-team.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/security.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/simplification.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/state-machines.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/targeted.md.json create mode 100644 test/fixtures/review-pins/files/review/specialists/testing.md.json create mode 100644 test/fixtures/review-pins/files/scripts/resolvers/fix-check.ts.json create mode 100644 test/fixtures/review-pins/files/ship/sections/adversarial.md.json create mode 100644 test/fixtures/review-pins/files/ship/sections/review-army.md.json create mode 100644 test/fixtures/review-pins/replays/3638.json create mode 100644 test/fixtures/review-pins/replays/G1.json create mode 100644 test/fixtures/review-pins/replays/GST-257.json create mode 100644 test/fixtures/review-pins/replays/GST-464.json create mode 100644 test/fixtures/review-pins/replays/GST-508+GST-464+GST-967.json create mode 100644 test/fixtures/review-pins/replays/GST-508+GST-464.json create mode 100644 test/fixtures/review-pins/replays/GST-508+R51+GST-911+GST-967.json create mode 100644 test/fixtures/review-pins/replays/GST-508+R51+GST-911.json create mode 100644 test/fixtures/review-pins/replays/GST-508+R51.json create mode 100644 test/fixtures/review-pins/replays/GST-508.json create mode 100644 test/fixtures/review-pins/replays/GST-850.json create mode 100644 test/fixtures/review-pins/replays/GST-898.json create mode 100644 test/fixtures/review-pins/replays/GST-911-redos.json create mode 100644 test/fixtures/review-pins/replays/GST-911.json create mode 100644 test/fixtures/review-pins/replays/GST-940.json create mode 100644 test/fixtures/review-pins/replays/GST-942.json create mode 100644 test/fixtures/review-pins/replays/GST-967.json create mode 100644 test/fixtures/review-pins/replays/GST-986.json create mode 100644 test/fixtures/review-pins/replays/GSTA-224-H1.json create mode 100644 test/fixtures/review-pins/replays/L.json create mode 100644 test/fixtures/review-pins/replays/PR-a.json create mode 100644 test/fixtures/review-pins/replays/R51.json create mode 100644 test/helpers/pin-merge-fixture.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bd1587d3..26c88c2bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.94.20.0] - 2026-11-03 + +### Changed + +- **The review pins and the harness pins are one file per pin, so PRs that repin different files no longer conflict (GST-1084).** The reviewer-brief pins that lived in one table in `test/review-packets-removed.test.ts` now live under `test/fixtures/review-pins/`: `files/.json` holds a file's sha and the replay that admits it, `replays/.json` holds one replay's written no-loss justification, and `brief-bodies/.json` holds one brief body's sha. The harness pins in `scripts/harness-version.json` moved to `scripts/harness-pins/.json`; the manifest keeps only the version and the patterns, so `HARNESS_VERSION` and its readers are unchanged. Before, two PRs that repinned adjacent entries conflicted on the shared table (at least five main-merge conflicts on 2026-11-02, two of them on #3852). Every entry is ported unchanged, including the combined GST-508 entries from #3852. + +### For contributors + +- `/ship`'s post-review route still counts a non-behavioral harness re-pin as release metadata (GST-1031): it now reads the pin files under `scripts/harness-pins/` and refuses a manifest that regains a `files` table. +- The checks are at least as strict as before. Every pin still needs its sha and a replay whose written justification names a no-loss replay or re-score, and the brief-body check still guards the bodies. A malformed or misnamed pin file, a stray file in a pin directory, or a `files` block back in `scripts/harness-version.json` now fails too. New tests prove that a stale pin still fails, and that two branches repinning different, adjacent pins merge cleanly with a real `git merge-tree`, while two repins of one pin still conflict. +- `bun run scripts/bump-harness-version.ts` writes and deletes pin files itself. Repin a review file by editing its `files/` pin and adding the replay file it names. + ## [1.94.19.0] - 2026-11-03 ### Fixed diff --git a/VERSION b/VERSION index 807c34fb9..f9f39ad26 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.19.0 +1.94.20.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 9468d6bf6..1c89466ae 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.19.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.20.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 4216c5256..38ce1e467 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1001,8 +1001,10 @@ paid test file, its fixtures and its skills' prompt files, including the generated SKILL.md) plus `HARNESS_VERSION`, keyed with model, Claude CLI version and policy version (`scripts/eval-trial-series.ts`). Each trial also records the full consumed-input fingerprint as provenance. `scripts/harness-version.json` -pins every shared-harness file (`GLOBAL_TOUCHFILES` plus the session runners, the -PTY harness, the e2e gate and the LLM judge) by blob hash. When +holds the version and the harness patterns, and `scripts/harness-pins/.json` +pins each shared-harness file (`GLOBAL_TOUCHFILES` plus the session runners, the +PTY harness, the e2e gate and the LLM judge) by blob hash, one file per pin, so +two PRs that record decisions for different harness files never conflict. When `test/harness-version.test.ts` fails, run one of: ```bash @@ -1010,8 +1012,8 @@ bun run scripts/bump-harness-version.ts --non-behavioral "" # new series for every case ``` -On a merge conflict in the manifest, take the higher version, then rerun the -script. +On a merge conflict in the manifest or a pin file, take the higher version, then +rerun the script. ## Ubicloud VMs (`bun run test:ubicloud`) diff --git a/docs/test-audit-2026-10.md b/docs/test-audit-2026-10.md index f1a4ba51e..fe8825438 100644 --- a/docs/test-audit-2026-10.md +++ b/docs/test-audit-2026-10.md @@ -50,14 +50,14 @@ per-failure-class breakdown from `test:health`. full consumed-input fingerprint (all touchfiles, globals included) as `series_fingerprint`, plus `harness_version`, as provenance. (`caseSeriesIdentitiesV2` in `scripts/eval-trial-series.ts`.) -- **Harness guard.** `scripts/harness-version.json` pins the blob of every - shared execution-harness file: `GLOBAL_TOUCHFILES` plus the session runners, - the PTY harness, the e2e gate and the LLM judge (36 files). A free test +- **Harness guard.** `scripts/harness-pins/.json` pins the blob of every + shared execution-harness file (`scripts/harness-version.json` keeps the version and patterns): `GLOBAL_TOUCHFILES` plus the session runners, + the PTY harness, the e2e gate and the LLM judge (42 files). A free test (`test/harness-version.test.ts`) fails when one changes without a decision. `bun run scripts/bump-harness-version.ts --bump ""` raises `HARNESS_VERSION` and starts a new series for every case; `--non-behavioral ""` records the new blobs as compatible with no - reset. Entries are one line per file so concurrent PRs that touch different + reset. Each pin is its own file so concurrent PRs that touch different files merge cleanly; on a conflict, take the higher version, then rerun. - **Weekly history (W4b).** Weeks for quarantine expiry and drift are `evals-periodic.yml` runs on `main`: scheduled runs and main dispatches. diff --git a/lib/ship-release.ts b/lib/ship-release.ts index 1cb9963fb..8a4575684 100644 --- a/lib/ship-release.ts +++ b/lib/ship-release.ts @@ -173,16 +173,20 @@ function commitRelease(cwd: string, release: string[], trailer: string | undefin /** * A harness pin edit that only records changed harness files as non-behavioral at the same version and patterns (GST-1031): release - * metadata, like a version stamp. A bump, a pattern change or an entry without its reason is behavior. + * metadata, like a version stamp. The pins live one file per harness file under scripts/harness-pins/ (GST-1084), beside a manifest + * holding only version and patterns. A bump, a pattern change or a pin without its reason is behavior. */ -const HARNESS_PIN = 'scripts/harness-version.json'; +const HARNESS_MANIFEST = 'scripts/harness-version.json'; +const HARNESS_PINS = 'scripts/harness-pins/'; function harnessRepin(cwd: string, from: string, to: string, p: string): boolean { - if (p !== HARNESS_PIN) return false; - const read = (tree: string): any => { const s = git(cwd, ['show', `${tree}:${p}`]); try { return s.status === 0 ? JSON.parse(s.stdout) : null; } catch { return null; } }; - const a = read(from), b = read(to); - if (!a?.files || !b?.files || typeof b.files !== 'object' || a.version !== b.version || JSON.stringify(a.patterns) !== JSON.stringify(b.patterns)) return false; - return Object.entries(b.files as Record).every(([f, e]) => JSON.stringify(a.files[f]) === JSON.stringify(e) - || (e?.since === b.version && typeof e?.nonBehavioral === 'string' && e.nonBehavioral.trim().length >= 10)); + if (p !== HARNESS_MANIFEST && !(p.startsWith(HARNESS_PINS) && p.endsWith('.json'))) return false; + const read = (tree: string, file: string): any => { const s = git(cwd, ['show', `${tree}:${file}`]); if (s.status !== 0) return undefined; try { return JSON.parse(s.stdout); } catch { return null; } }; + const a = read(from, HARNESS_MANIFEST), b = read(to, HARNESS_MANIFEST); + if (!a || !b || Object.keys(b).sort().join() !== 'patterns,version' || a.version !== b.version || JSON.stringify(a.patterns) !== JSON.stringify(b.patterns)) return false; + if (p === HARNESS_MANIFEST) return true; + const e = read(to, p); + return e === undefined || JSON.stringify(read(from, p)) === JSON.stringify(e) + || (e?.since === b.version && typeof e?.nonBehavioral === 'string' && e.nonBehavioral.trim().length >= 10); } /** diff --git a/package.json b/package.json index d6c71b54c..436bce3c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.19", + "version": "1.94.20", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/bump-harness-version.ts b/scripts/bump-harness-version.ts index b15a5336c..7f7d7d36a 100644 --- a/scripts/bump-harness-version.ts +++ b/scripts/bump-harness-version.ts @@ -1,8 +1,9 @@ #!/usr/bin/env -S bun --no-env-file --config=/dev/null /** * Keep HARNESS_VERSION honest (CEO-27/ENG-3/DX-6). scripts/harness-version.json - * pins the blob of every shared-harness file (GLOBAL_TOUCHFILES plus its - * `patterns`). test/harness-version.test.ts fails when a harness file changes + * holds the version and the harness `patterns`; scripts/harness-pins/.json + * pins the blob of every shared-harness file (GLOBAL_TOUCHFILES plus those + * patterns). test/harness-version.test.ts fails when a harness file changes * without a decision, and this script records the decision: * * --bump "" the change alters how cases behave: version + 1, @@ -12,9 +13,10 @@ * compatible and no series resets * --check print drift and the fix; exit 1 on drift * - * Entries are one line per file, so two PRs touching different harness files - * merge cleanly. When both bump, or both touch one file, git conflicts on the - * manifest: take the higher version, then rerun this script. + * Each pin is its own file (GST-1084), so two PRs touching different harness + * files never edit the same file and merge cleanly. When both bump, or both + * touch one file, git conflicts on the manifest or that pin: take the higher + * version, then rerun this script. */ import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -23,14 +25,35 @@ import { harnessFiles, treeEntries, type TreeEntry } from './eval-trial-series'; const ROOT = path.resolve(import.meta.dir, '..'); export const MANIFEST = 'scripts/harness-version.json'; +export const PINS_DIR = 'scripts/harness-pins'; export const DOC_ANCHOR = 'docs/TESTING_INTERNALS.md#harness-version'; export interface ManifestEntry { blob: string; since: number; nonBehavioral?: string } export interface HarnessManifest { version: number; patterns: string[]; files: Record } export interface Drift { changed: string[]; added: string[]; removed: string[] } +const pinPath = (root: string, file: string) => path.join(root, PINS_DIR, `${file}.json`); + +/** The pin files under root's PINS_DIR, keyed by pinned path; any file that is not .json is an error. */ +function readPins(root: string): Record { + const dir = path.join(root, PINS_DIR); + const files: Record = {}; + for (const rel of (fs.readdirSync(dir, { recursive: true }) as string[]).map(f => f.split(path.sep).join('/')).sort()) { + if (fs.statSync(path.join(dir, rel)).isDirectory()) continue; + if (!rel.endsWith('.json')) throw new Error(`${PINS_DIR}/${rel}: every harness pin is .json`); + try { + files[rel.slice(0, -'.json'.length)] = JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); + } catch (e) { + throw new Error(`${PINS_DIR}/${rel}: ${(e as Error).message}`); + } + } + return files; +} + export function readManifest(root = ROOT): HarnessManifest { - return JSON.parse(fs.readFileSync(path.join(root, MANIFEST), 'utf8')); + const head = JSON.parse(fs.readFileSync(path.join(root, MANIFEST), 'utf8')); + if (Object.keys(head).sort().join() !== 'patterns,version') throw new Error(`${MANIFEST} holds only version and patterns; pins live in ${PINS_DIR}/.json`); + return { version: head.version, patterns: head.patterns, files: readPins(root) }; } /** The tracked harness files with blobs of their working-tree bytes (an unstaged edit counts; a deleted file drops out). */ @@ -43,11 +66,11 @@ export function workingHarnessFiles(root: string, patterns: readonly string[]): return tracked.map((entry, index) => ({ file: entry.file, blob: blobs[index]! })); } -/** Harness files whose blob differs from, is missing from, or no longer matches an entry in the manifest. */ +/** Harness files whose blob differs from, is missing from, or no longer matches an entry in the manifest; a pin ahead of the manifest version (a bump that failed partway) counts as changed. */ export function harnessDrift(manifest: HarnessManifest, files: TreeEntry[]): Drift { const current = new Map(files.map(entry => [entry.file, entry.blob])); return { - changed: files.filter(entry => manifest.files[entry.file] && manifest.files[entry.file]!.blob !== entry.blob).map(entry => entry.file).sort(), + changed: files.filter(entry => manifest.files[entry.file] && (manifest.files[entry.file]!.blob !== entry.blob || manifest.files[entry.file]!.since > manifest.version)).map(entry => entry.file).sort(), added: files.filter(entry => !manifest.files[entry.file]).map(entry => entry.file).sort(), removed: Object.keys(manifest.files).filter(file => !current.has(file)).sort(), }; @@ -64,7 +87,7 @@ export function formatDrift(drift: Drift, version: number): string { 'Fix (pick one):', ' bun run scripts/bump-harness-version.ts --non-behavioral "" # refactor/comment/logging: no series reset', ' bun run scripts/bump-harness-version.ts --bump "" # starts a new pass-rate series for every case', - `Merge conflict in ${MANIFEST}: take the higher version, then rerun the script. See ${DOC_ANCHOR}.`, + `Merge conflict in ${MANIFEST} or ${PINS_DIR}/: take the higher version, then rerun the script. See ${DOC_ANCHOR}.`, ].join('\n'); } @@ -82,11 +105,16 @@ export function decide(manifest: HarnessManifest, files: TreeEntry[], decision: : manifest.files[entry.file]!])) }; } -/** Stable, one-entry-per-line JSON so concurrent edits to different files merge cleanly. */ -export function serializeManifest(manifest: HarnessManifest): string { - const files = Object.keys(manifest.files).sort().map(file => ` ${JSON.stringify(file)}: ${JSON.stringify(manifest.files[file])}`); - return `{\n "version": ${manifest.version},\n "patterns": [\n${manifest.patterns.map(pattern => ` ${JSON.stringify(pattern)}`).join(',\n')}\n ],\n` - + ` "files": {\n${files.join(',\n')}\n }\n}\n`; +/** Write one pin file per harness file (rewriting only pins whose bytes change, deleting dropped ones), then the manifest head, so a failed pin write never moves the version. */ +export function writeManifest(root: string, manifest: HarnessManifest): void { + for (const file of Object.keys(readPins(root))) if (!manifest.files[file]) fs.rmSync(pinPath(root, file)); + for (const [file, entry] of Object.entries(manifest.files)) { + const target = pinPath(root, file), text = `${JSON.stringify(entry)}\n`; + if (fs.existsSync(target) && fs.readFileSync(target, 'utf8') === text) continue; + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, text); + } + fs.writeFileSync(path.join(root, MANIFEST), `${JSON.stringify({ version: manifest.version, patterns: manifest.patterns }, null, 2)}\n`); } if (import.meta.main) { @@ -106,7 +134,7 @@ if (import.meta.main) { } if (mode === '--non-behavioral' && !hasDrift(drift)) { console.log('no harness drift: nothing to record'); process.exit(0); } const next = decide(manifest, files, mode === '--bump' ? 'bump' : 'non-behavioral', argv[1] ?? ''); - fs.writeFileSync(path.join(ROOT, MANIFEST), serializeManifest(next)); + writeManifest(ROOT, next); console.log(mode === '--bump' ? `HARNESS_VERSION ${manifest.version} -> ${next.version}: every case starts a new pass-rate series (reason: ${argv[1]})` : `recorded ${drift.changed.length + drift.added.length} file(s) as non-behavioral at HARNESS_VERSION ${next.version}${drift.removed.length ? `, dropped ${drift.removed.length} removed file(s)` : ''}`); diff --git a/scripts/harness-pins/scripts/lib/paid-cases.ts.json b/scripts/harness-pins/scripts/lib/paid-cases.ts.json new file mode 100644 index 000000000..be046eaa0 --- /dev/null +++ b/scripts/harness-pins/scripts/lib/paid-cases.ts.json @@ -0,0 +1 @@ +{"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"} diff --git a/scripts/harness-pins/scripts/lib/paid-plan.ts.json b/scripts/harness-pins/scripts/lib/paid-plan.ts.json new file mode 100644 index 000000000..800cea9ce --- /dev/null +++ b/scripts/harness-pins/scripts/lib/paid-plan.ts.json @@ -0,0 +1 @@ +{"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"} diff --git a/scripts/harness-pins/scripts/lib/paid-report.ts.json b/scripts/harness-pins/scripts/lib/paid-report.ts.json new file mode 100644 index 000000000..643e38909 --- /dev/null +++ b/scripts/harness-pins/scripts/lib/paid-report.ts.json @@ -0,0 +1 @@ +{"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"} diff --git a/scripts/harness-pins/scripts/lib/published-text.ts.json b/scripts/harness-pins/scripts/lib/published-text.ts.json new file mode 100644 index 000000000..5596a3fed --- /dev/null +++ b/scripts/harness-pins/scripts/lib/published-text.ts.json @@ -0,0 +1 @@ +{"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6} diff --git a/scripts/harness-pins/scripts/lib/shard-engine.ts.json b/scripts/harness-pins/scripts/lib/shard-engine.ts.json new file mode 100644 index 000000000..84495fc10 --- /dev/null +++ b/scripts/harness-pins/scripts/lib/shard-engine.ts.json @@ -0,0 +1 @@ +{"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"} diff --git a/scripts/harness-pins/scripts/test-paid-shards.ts.json b/scripts/harness-pins/scripts/test-paid-shards.ts.json new file mode 100644 index 000000000..b15d7b865 --- /dev/null +++ b/scripts/harness-pins/scripts/test-paid-shards.ts.json @@ -0,0 +1 @@ +{"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"} diff --git a/scripts/harness-pins/scripts/test-pr-profile.ts.json b/scripts/harness-pins/scripts/test-pr-profile.ts.json new file mode 100644 index 000000000..1a838ffba --- /dev/null +++ b/scripts/harness-pins/scripts/test-pr-profile.ts.json @@ -0,0 +1 @@ +{"blob":"b552b8825d693493decef76b05b976858cab087e","since":6,"nonBehavioral":"test-pr-profile.ts lists .github/workflows/windows-scoped-tests.yml as free-lane only; the workflow runs no paid case, so no case selection or verdict can change"} diff --git a/scripts/harness-pins/scripts/test-strict-output.ts.json b/scripts/harness-pins/scripts/test-strict-output.ts.json new file mode 100644 index 000000000..3678cec32 --- /dev/null +++ b/scripts/harness-pins/scripts/test-strict-output.ts.json @@ -0,0 +1 @@ +{"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6} diff --git a/scripts/harness-pins/test/helpers/agent-sdk-runner.ts.json b/scripts/harness-pins/test/helpers/agent-sdk-runner.ts.json new file mode 100644 index 000000000..27a5c6a18 --- /dev/null +++ b/scripts/harness-pins/test/helpers/agent-sdk-runner.ts.json @@ -0,0 +1 @@ +{"blob":"d66dac458b8c13250973e550edf8087d4044cd0b","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets. Also: runAgentSdkTest now validates systemPrompt before starting; every live caller already passed the claude_code preset, so the prompt the model sees is unchanged and no verdict can change"} diff --git a/scripts/harness-pins/test/helpers/auq-substance-panel.ts.json b/scripts/harness-pins/test/helpers/auq-substance-panel.ts.json new file mode 100644 index 000000000..405936de5 --- /dev/null +++ b/scripts/harness-pins/test/helpers/auq-substance-panel.ts.json @@ -0,0 +1 @@ +{"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6} diff --git a/scripts/harness-pins/test/helpers/claude-pty-runner.ts.json b/scripts/harness-pins/test/helpers/claude-pty-runner.ts.json new file mode 100644 index 000000000..5a1ad1203 --- /dev/null +++ b/scripts/harness-pins/test/helpers/claude-pty-runner.ts.json @@ -0,0 +1 @@ +{"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."} diff --git a/scripts/harness-pins/test/helpers/codex-host-runner.ts.json b/scripts/harness-pins/test/helpers/codex-host-runner.ts.json new file mode 100644 index 000000000..9888d6845 --- /dev/null +++ b/scripts/harness-pins/test/helpers/codex-host-runner.ts.json @@ -0,0 +1 @@ +{"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"} diff --git a/scripts/harness-pins/test/helpers/codex-session-runner.ts.json b/scripts/harness-pins/test/helpers/codex-session-runner.ts.json new file mode 100644 index 000000000..9946b5fa1 --- /dev/null +++ b/scripts/harness-pins/test/helpers/codex-session-runner.ts.json @@ -0,0 +1 @@ +{"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"} diff --git a/scripts/harness-pins/test/helpers/e2e-gate.ts.json b/scripts/harness-pins/test/helpers/e2e-gate.ts.json new file mode 100644 index 000000000..e7bc9cd43 --- /dev/null +++ b/scripts/harness-pins/test/helpers/e2e-gate.ts.json @@ -0,0 +1 @@ +{"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6} diff --git a/scripts/harness-pins/test/helpers/e2e-helpers.ts.json b/scripts/harness-pins/test/helpers/e2e-helpers.ts.json new file mode 100644 index 000000000..26fcdf739 --- /dev/null +++ b/scripts/harness-pins/test/helpers/e2e-helpers.ts.json @@ -0,0 +1 @@ +{"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"} diff --git a/scripts/harness-pins/test/helpers/eval-budgets.ts.json b/scripts/harness-pins/test/helpers/eval-budgets.ts.json new file mode 100644 index 000000000..914ab65fe --- /dev/null +++ b/scripts/harness-pins/test/helpers/eval-budgets.ts.json @@ -0,0 +1 @@ +{"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6} diff --git a/scripts/harness-pins/test/helpers/eval-store.ts.json b/scripts/harness-pins/test/helpers/eval-store.ts.json new file mode 100644 index 000000000..d81bec0f5 --- /dev/null +++ b/scripts/harness-pins/test/helpers/eval-store.ts.json @@ -0,0 +1 @@ +{"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"} diff --git a/scripts/harness-pins/test/helpers/hermetic-env.ts.json b/scripts/harness-pins/test/helpers/hermetic-env.ts.json new file mode 100644 index 000000000..10202734c --- /dev/null +++ b/scripts/harness-pins/test/helpers/hermetic-env.ts.json @@ -0,0 +1 @@ +{"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6} diff --git a/scripts/harness-pins/test/helpers/llm-judge.ts.json b/scripts/harness-pins/test/helpers/llm-judge.ts.json new file mode 100644 index 000000000..c33f39919 --- /dev/null +++ b/scripts/harness-pins/test/helpers/llm-judge.ts.json @@ -0,0 +1 @@ +{"blob":"ff32d03f32e4af76c6c7314a79cd1b35263ed0dd","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged. GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading so the review E2E cases extract the same section (now the skill's own reworded bytes); llm-judge.ts changes one bullet of the cross-skill consistency judge's intended architecture to name review-bot-history.md beside the older greptile-history.md name, so its calibration corpus (greptile-history inputs) stays valid. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own series"} diff --git a/scripts/harness-pins/test/helpers/paid-test-set.ts.json b/scripts/harness-pins/test/helpers/paid-test-set.ts.json new file mode 100644 index 000000000..103260b3a --- /dev/null +++ b/scripts/harness-pins/test/helpers/paid-test-set.ts.json @@ -0,0 +1 @@ +{"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6} diff --git a/scripts/harness-pins/test/helpers/pty-screen.ts.json b/scripts/harness-pins/test/helpers/pty-screen.ts.json new file mode 100644 index 000000000..0f8f53af1 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty-screen.ts.json @@ -0,0 +1 @@ +{"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/auq.ts.json b/scripts/harness-pins/test/helpers/pty/auq.ts.json new file mode 100644 index 000000000..64b0f232d --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/auq.ts.json @@ -0,0 +1 @@ +{"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/binary.ts.json b/scripts/harness-pins/test/helpers/pty/binary.ts.json new file mode 100644 index 000000000..34e957bbd --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/binary.ts.json @@ -0,0 +1 @@ +{"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/boundaries.ts.json b/scripts/harness-pins/test/helpers/pty/boundaries.ts.json new file mode 100644 index 000000000..fe58172e7 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/boundaries.ts.json @@ -0,0 +1 @@ +{"blob":"c09b69611fb22ddf093adb2baa83434494e5f8fa","since":6,"nonBehavioral":"Widens the two plan-eng-review boundary predicates' qid regex to also accept the optional ;signal= hint suffix (#3442). Only a question carrying that suffix can change verdict, and only the changed question-tuning resolver asks for it; every case that runs a skill with that resolver owns its generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."} diff --git a/scripts/harness-pins/test/helpers/pty/classify.ts.json b/scripts/harness-pins/test/helpers/pty/classify.ts.json new file mode 100644 index 000000000..4216ea1f3 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/classify.ts.json @@ -0,0 +1 @@ +{"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/fake-session.ts.json b/scripts/harness-pins/test/helpers/pty/fake-session.ts.json new file mode 100644 index 000000000..3491ae558 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/fake-session.ts.json @@ -0,0 +1 @@ +{"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/judge.ts.json b/scripts/harness-pins/test/helpers/pty/judge.ts.json new file mode 100644 index 000000000..afe888646 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/judge.ts.json @@ -0,0 +1 @@ +{"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/launch.ts.json b/scripts/harness-pins/test/helpers/pty/launch.ts.json new file mode 100644 index 000000000..a688b12bb --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/launch.ts.json @@ -0,0 +1 @@ +{"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"} diff --git a/scripts/harness-pins/test/helpers/pty/plan-native.ts.json b/scripts/harness-pins/test/helpers/pty/plan-native.ts.json new file mode 100644 index 000000000..14c809f04 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/plan-native.ts.json @@ -0,0 +1 @@ +{"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/runners/counting.ts.json b/scripts/harness-pins/test/helpers/pty/runners/counting.ts.json new file mode 100644 index 000000000..51764e7e4 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/runners/counting.ts.json @@ -0,0 +1 @@ +{"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/runners/floor.ts.json b/scripts/harness-pins/test/helpers/pty/runners/floor.ts.json new file mode 100644 index 000000000..e70ba480c --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/runners/floor.ts.json @@ -0,0 +1 @@ +{"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"} diff --git a/scripts/harness-pins/test/helpers/pty/runners/observation.ts.json b/scripts/harness-pins/test/helpers/pty/runners/observation.ts.json new file mode 100644 index 000000000..024bdf6f3 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/runners/observation.ts.json @@ -0,0 +1 @@ +{"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6} diff --git a/scripts/harness-pins/test/helpers/pty/screen.ts.json b/scripts/harness-pins/test/helpers/pty/screen.ts.json new file mode 100644 index 000000000..1b8e1e677 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/screen.ts.json @@ -0,0 +1 @@ +{"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"} diff --git a/scripts/harness-pins/test/helpers/pty/session.ts.json b/scripts/harness-pins/test/helpers/pty/session.ts.json new file mode 100644 index 000000000..81322011d --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/session.ts.json @@ -0,0 +1 @@ +{"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"} diff --git a/scripts/harness-pins/test/helpers/pty/transcript-cost.ts.json b/scripts/harness-pins/test/helpers/pty/transcript-cost.ts.json new file mode 100644 index 000000000..346bc89e8 --- /dev/null +++ b/scripts/harness-pins/test/helpers/pty/transcript-cost.ts.json @@ -0,0 +1 @@ +{"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"} diff --git a/scripts/harness-pins/test/helpers/session-drain-policy.ts.json b/scripts/harness-pins/test/helpers/session-drain-policy.ts.json new file mode 100644 index 000000000..6170d5af4 --- /dev/null +++ b/scripts/harness-pins/test/helpers/session-drain-policy.ts.json @@ -0,0 +1 @@ +{"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6} diff --git a/scripts/harness-pins/test/helpers/session-ledger.ts.json b/scripts/harness-pins/test/helpers/session-ledger.ts.json new file mode 100644 index 000000000..b1b7d302c --- /dev/null +++ b/scripts/harness-pins/test/helpers/session-ledger.ts.json @@ -0,0 +1 @@ +{"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"} diff --git a/scripts/harness-pins/test/helpers/session-runner.ts.json b/scripts/harness-pins/test/helpers/session-runner.ts.json new file mode 100644 index 000000000..24e92def6 --- /dev/null +++ b/scripts/harness-pins/test/helpers/session-runner.ts.json @@ -0,0 +1 @@ +{"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"} diff --git a/scripts/harness-pins/test/helpers/skill-fixture.ts.json b/scripts/harness-pins/test/helpers/skill-fixture.ts.json new file mode 100644 index 000000000..a6731990a --- /dev/null +++ b/scripts/harness-pins/test/helpers/skill-fixture.ts.json @@ -0,0 +1 @@ +{"blob":"ce7609eacc7a8265a4a211358d1b4fb49ea85bc5","since":6,"nonBehavioral":"GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading ('Check for review bot comments') so the review E2E cases extract the same section, now the skill's own reworded bytes; llm-judge.ts updates only the cross-skill consistency judge's intended architecture to the renamed review-bot-history.md and its record helper. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own new series"} diff --git a/scripts/harness-pins/test/helpers/skill-startup.ts.json b/scripts/harness-pins/test/helpers/skill-startup.ts.json new file mode 100644 index 000000000..17647bd7a --- /dev/null +++ b/scripts/harness-pins/test/helpers/skill-startup.ts.json @@ -0,0 +1 @@ +{"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"} diff --git a/scripts/harness-pins/test/helpers/test-selection.ts.json b/scripts/harness-pins/test/helpers/test-selection.ts.json new file mode 100644 index 000000000..cce9f6a3c --- /dev/null +++ b/scripts/harness-pins/test/helpers/test-selection.ts.json @@ -0,0 +1 @@ +{"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"} diff --git a/scripts/harness-pins/test/helpers/touchfiles.ts.json b/scripts/harness-pins/test/helpers/touchfiles.ts.json new file mode 100644 index 000000000..913f9600d --- /dev/null +++ b/scripts/harness-pins/test/helpers/touchfiles.ts.json @@ -0,0 +1 @@ +{"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} diff --git a/scripts/harness-version.json b/scripts/harness-version.json index c262bd9aa..a083d2d17 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -9,49 +9,5 @@ "test/helpers/pty-screen.ts", "test/helpers/e2e-gate.ts", "test/helpers/llm-judge.ts" - ], - "files": { - "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, - "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"}, - "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, - "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, - "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "scripts/test-pr-profile.ts": {"blob":"b552b8825d693493decef76b05b976858cab087e","since":6,"nonBehavioral":"test-pr-profile.ts lists .github/workflows/windows-scoped-tests.yml as free-lane only; the workflow runs no paid case, so no case selection or verdict can change"}, - "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, - "test/helpers/agent-sdk-runner.ts": {"blob":"d66dac458b8c13250973e550edf8087d4044cd0b","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets. Also: runAgentSdkTest now validates systemPrompt before starting; every live caller already passed the claude_code preset, so the prompt the model sees is unchanged and no verdict can change"}, - "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, - "test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, - "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, - "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, - "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, - "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, - "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, - "test/helpers/llm-judge.ts": {"blob":"ff32d03f32e4af76c6c7314a79cd1b35263ed0dd","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged. GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading so the review E2E cases extract the same section (now the skill's own reworded bytes); llm-judge.ts changes one bullet of the cross-skill consistency judge's intended architecture to name review-bot-history.md beside the older greptile-history.md name, so its calibration corpus (greptile-history inputs) stays valid. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own series"}, - "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, - "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6}, - "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6}, - "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6}, - "test/helpers/pty/boundaries.ts": {"blob":"c09b69611fb22ddf093adb2baa83434494e5f8fa","since":6,"nonBehavioral":"Widens the two plan-eng-review boundary predicates' qid regex to also accept the optional ;signal= hint suffix (#3442). Only a question carrying that suffix can change verdict, and only the changed question-tuning resolver asks for it; every case that runs a skill with that resolver owns its generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."}, - "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, - "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, - "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, - "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, - "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, - "test/helpers/pty/runners/floor.ts": {"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"}, - "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, - "test/helpers/pty/screen.ts": {"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"}, - "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, - "test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, - "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, - "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, - "test/helpers/skill-fixture.ts": {"blob":"ce7609eacc7a8265a4a211358d1b4fb49ea85bc5","since":6,"nonBehavioral":"GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading ('Check for review bot comments') so the review E2E cases extract the same section, now the skill's own reworded bytes; llm-judge.ts updates only the cross-skill consistency judge's intended architecture to the renamed review-bot-history.md and its record helper. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own new series"}, - "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"}, - "test/helpers/test-selection.ts": {"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"}, - "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} - } + ] } diff --git a/test/fixtures/review-pins/brief-bodies/fixtures.json b/test/fixtures/review-pins/brief-bodies/fixtures.json new file mode 100644 index 000000000..e8676007c --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/fixtures.json @@ -0,0 +1 @@ +{"sha":"fbe52f3a9e0f4442eb755f1747a95944b373b439bd05c61f3eb68345d1061a5b"} diff --git a/test/fixtures/review-pins/brief-bodies/framing.json b/test/fixtures/review-pins/brief-bodies/framing.json new file mode 100644 index 000000000..dabdf42b7 --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/framing.json @@ -0,0 +1 @@ +{"sha":"aad667b5ea4c6d23d930dafffa38178c04be388cd4364fb73759fc92af17bb63"} diff --git a/test/fixtures/review-pins/brief-bodies/native.json b/test/fixtures/review-pins/brief-bodies/native.json new file mode 100644 index 000000000..8b43a986f --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/native.json @@ -0,0 +1 @@ +{"sha":"a7cc08fb31b6ca65932a316d589f0b09677426d193d63f88ec74fee6d93ce053"} diff --git a/test/fixtures/review-pins/brief-bodies/outside.json b/test/fixtures/review-pins/brief-bodies/outside.json new file mode 100644 index 000000000..9cdf30705 --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/outside.json @@ -0,0 +1 @@ +{"sha":"bd96406da28b03742384560425a333b6d6d7ed2be95bd2026c90e0039632cee2"} diff --git a/test/fixtures/review-pins/brief-bodies/redTeam.json b/test/fixtures/review-pins/brief-bodies/redTeam.json new file mode 100644 index 000000000..32c0b6cbf --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/redTeam.json @@ -0,0 +1 @@ +{"sha":"6dd670ea040535ca6cceab81c29a395bb67ce5a2ab8fa2b81e51bc4711e4aada"} diff --git a/test/fixtures/review-pins/brief-bodies/specialist.json b/test/fixtures/review-pins/brief-bodies/specialist.json new file mode 100644 index 000000000..8be6a67c2 --- /dev/null +++ b/test/fixtures/review-pins/brief-bodies/specialist.json @@ -0,0 +1 @@ +{"sha":"1ebe47ddb3291a9e4fe90b5a260eb57e328144959549bc95c119b68d23652cb5"} diff --git a/test/fixtures/review-pins/files/lib/review-depth.ts.json b/test/fixtures/review-pins/files/lib/review-depth.ts.json new file mode 100644 index 000000000..aa7fee0ec --- /dev/null +++ b/test/fixtures/review-pins/files/lib/review-depth.ts.json @@ -0,0 +1 @@ +{"sha":"25050a7a61f81df649d1df0d0237dbf961671356395317c6a527741f5bb56877","replay":"GST-940"} diff --git a/test/fixtures/review-pins/files/lib/review-risk.ts.json b/test/fixtures/review-pins/files/lib/review-risk.ts.json new file mode 100644 index 000000000..d3a277ac6 --- /dev/null +++ b/test/fixtures/review-pins/files/lib/review-risk.ts.json @@ -0,0 +1 @@ +{"sha":"08464b78a723ee463ca3a28d0abbc4eb8f4f512192061c703743707b66b5a311","replay":"GST-942"} diff --git a/test/fixtures/review-pins/files/review/checklist.md.json b/test/fixtures/review-pins/files/review/checklist.md.json new file mode 100644 index 000000000..64060cc55 --- /dev/null +++ b/test/fixtures/review-pins/files/review/checklist.md.json @@ -0,0 +1 @@ +{"sha":"a32963ef38d9d3613c58c805e8517cd2e593c63769b30dd7d40e2592fa48fc78","replay":"GSTA-224 H1"} diff --git a/test/fixtures/review-pins/files/review/design-checklist.md.json b/test/fixtures/review-pins/files/review/design-checklist.md.json new file mode 100644 index 000000000..f3b8e1534 --- /dev/null +++ b/test/fixtures/review-pins/files/review/design-checklist.md.json @@ -0,0 +1 @@ +{"sha":"b9205f88313cc64902c0b13e9685722ab8138d3dcc58916154f47274abda7d83","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/gate-integrity.md.json b/test/fixtures/review-pins/files/review/gate-integrity.md.json new file mode 100644 index 000000000..7586b3133 --- /dev/null +++ b/test/fixtures/review-pins/files/review/gate-integrity.md.json @@ -0,0 +1 @@ +{"sha":"f570db344c9a01df633bcaa281d6017f4f12420fd3e0aa3078c1ce06e0df8332","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/sections/adversarial.md.json b/test/fixtures/review-pins/files/review/sections/adversarial.md.json new file mode 100644 index 000000000..39bc50c43 --- /dev/null +++ b/test/fixtures/review-pins/files/review/sections/adversarial.md.json @@ -0,0 +1 @@ +{"sha":"e6962fa25b78f362654f2954f79683e524d2ebd452d1494f8a2b099888e8e255","replay":"G1"} diff --git a/test/fixtures/review-pins/files/review/sections/review-army.md.json b/test/fixtures/review-pins/files/review/sections/review-army.md.json new file mode 100644 index 000000000..2115bd9d7 --- /dev/null +++ b/test/fixtures/review-pins/files/review/sections/review-army.md.json @@ -0,0 +1 @@ +{"sha":"78329b64640260cd8831ca81a9189f3f8dcade9c406431631d3f2cd798cd3967","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/api-contract.md.json b/test/fixtures/review-pins/files/review/specialists/api-contract.md.json new file mode 100644 index 000000000..d03e20c76 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/api-contract.md.json @@ -0,0 +1 @@ +{"sha":"263d23ac119dd601d315c191dbdbd503d47c00264c9c0ca81959559ca11d4e95","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/data-migration.md.json b/test/fixtures/review-pins/files/review/specialists/data-migration.md.json new file mode 100644 index 000000000..a240f5f6d --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/data-migration.md.json @@ -0,0 +1 @@ +{"sha":"b6fd9eb229002ea598f8fe9ff53b1cd8821e3bd37a7aa7b07b5526556c71ebca","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/maintainability.md.json b/test/fixtures/review-pins/files/review/specialists/maintainability.md.json new file mode 100644 index 000000000..a4d0f4a46 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/maintainability.md.json @@ -0,0 +1 @@ +{"sha":"f8015d549e70297f782f0aabf6fe3a9fe31ec5ed61c311ab854e4a858282ed9b","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/performance.md.json b/test/fixtures/review-pins/files/review/specialists/performance.md.json new file mode 100644 index 000000000..c614d313c --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/performance.md.json @@ -0,0 +1 @@ +{"sha":"545c294ae53638b4c8524e8cde08246a4ce3b5c287ec7c44e27f5556a3b0e8cc","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/red-team.md.json b/test/fixtures/review-pins/files/review/specialists/red-team.md.json new file mode 100644 index 000000000..fa156d736 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/red-team.md.json @@ -0,0 +1 @@ +{"sha":"9ea05149f5b13d6a19ecec26285142d09d259cf80a515af828abdd1f43320427","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/security.md.json b/test/fixtures/review-pins/files/review/specialists/security.md.json new file mode 100644 index 000000000..16f65b60e --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/security.md.json @@ -0,0 +1 @@ +{"sha":"17bfe813eb3de22835af048ec359770720f1d7f371a26460169f7e32b1bcac9f","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/simplification.md.json b/test/fixtures/review-pins/files/review/specialists/simplification.md.json new file mode 100644 index 000000000..2adc034c8 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/simplification.md.json @@ -0,0 +1 @@ +{"sha":"f33846b1dcf2fdeef82d0744ab2ef48fb4a33b34a737dafc33dd0e891756689f","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/state-machines.md.json b/test/fixtures/review-pins/files/review/specialists/state-machines.md.json new file mode 100644 index 000000000..8584a4b6e --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/state-machines.md.json @@ -0,0 +1 @@ +{"sha":"597215e4438d85d9cc71b48097c61eef7cba4f4b09dc14c92b4b94893b3d3ea2","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/review/specialists/targeted.md.json b/test/fixtures/review-pins/files/review/specialists/targeted.md.json new file mode 100644 index 000000000..ec05c6178 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/targeted.md.json @@ -0,0 +1 @@ +{"sha":"7f92a0d0a8319c3c0e076fb3b1a4689c45120b2be2c14dfc7d4cb03aafa9d6a5","replay":"GST-257"} diff --git a/test/fixtures/review-pins/files/review/specialists/testing.md.json b/test/fixtures/review-pins/files/review/specialists/testing.md.json new file mode 100644 index 000000000..995b63a34 --- /dev/null +++ b/test/fixtures/review-pins/files/review/specialists/testing.md.json @@ -0,0 +1 @@ +{"sha":"d6cc2bef7be562387d0c0a22fefe646dff9c295f1d34d8453a6cb496584150e4","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/scripts/resolvers/fix-check.ts.json b/test/fixtures/review-pins/files/scripts/resolvers/fix-check.ts.json new file mode 100644 index 000000000..37eca24b7 --- /dev/null +++ b/test/fixtures/review-pins/files/scripts/resolvers/fix-check.ts.json @@ -0,0 +1 @@ +{"sha":"712bf5b0c439e96bac32f19cbe25394dabb4c94aa66600100bfde9b100e15a08","replay":"#3638"} diff --git a/test/fixtures/review-pins/files/ship/sections/adversarial.md.json b/test/fixtures/review-pins/files/ship/sections/adversarial.md.json new file mode 100644 index 000000000..98d68b7ae --- /dev/null +++ b/test/fixtures/review-pins/files/ship/sections/adversarial.md.json @@ -0,0 +1 @@ +{"sha":"0142d6bc3f2a5bceb9f8b2adef1d3a206e5836c99a0667fe995568da2e80a5a6","replay":"GST-508+GST-464+GST-967"} diff --git a/test/fixtures/review-pins/files/ship/sections/review-army.md.json b/test/fixtures/review-pins/files/ship/sections/review-army.md.json new file mode 100644 index 000000000..73517e9fe --- /dev/null +++ b/test/fixtures/review-pins/files/ship/sections/review-army.md.json @@ -0,0 +1 @@ +{"sha":"de3e0d35f2af56492c42fd287b6868fbe19bce96b86321e74a4947ba909b74eb","replay":"GST-508+GST-464+GST-967"} diff --git a/test/fixtures/review-pins/replays/3638.json b/test/fixtures/review-pins/replays/3638.json new file mode 100644 index 000000000..3f012cc08 --- /dev/null +++ b/test/fixtures/review-pins/replays/3638.json @@ -0,0 +1 @@ +{"replay":"#3638","justification":"no-loss round 3: main restored the pre-#3621 reviewers (v1.91.331.0)"} diff --git a/test/fixtures/review-pins/replays/G1.json b/test/fixtures/review-pins/replays/G1.json new file mode 100644 index 000000000..0327449a8 --- /dev/null +++ b/test/fixtures/review-pins/replays/G1.json @@ -0,0 +1 @@ +{"replay":"G1","justification":"GSTA-167 variant G Arm G1: live no-loss replay of both prose voices with part 1's prompts, 2 passes, normalizer v2, paired rule against pooled C' (parts 1 and 3; specialists at medium as in C and C2)"} diff --git a/test/fixtures/review-pins/replays/GST-257.json b/test/fixtures/review-pins/replays/GST-257.json new file mode 100644 index 000000000..dfb1e6c7f --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-257.json @@ -0,0 +1 @@ +{"replay":"GST-257","justification":"GST-257 PR 1 dispatch-superset no-loss check (test/review-targeted.test.ts): on every fixture diff the branch dispatches main's reviewers at 8be7f21 unchanged plus only the targeted reader, and over the last 100 non-release commits on main it lost none; adds the targeted role and brief to the G1 and L bytes, whose existing reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-464.json b/test/fixtures/review-pins/replays/GST-464.json new file mode 100644 index 000000000..4ebbfe348 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-464.json @@ -0,0 +1 @@ +{"replay":"GST-464","justification":"GST-464 brief-identity no-loss check (the brief-body test below): every reviewer brief lib/review-brief.ts cuts from ship/sections/review-army.md and adversarial.md is byte-identical to the bodies at the L and GST-257 pins; only lead instructions outside the cut spans changed"} diff --git a/test/fixtures/review-pins/replays/GST-508+GST-464+GST-967.json b/test/fixtures/review-pins/replays/GST-508+GST-464+GST-967.json new file mode 100644 index 000000000..2a5444764 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508+GST-464+GST-967.json @@ -0,0 +1 @@ +{"replay":"GST-508+GST-464+GST-967","justification":"wave review-53cfb5bfe011 merge of main 68410828f into #3852: ship/sections/review-army.md and adversarial.md are main's GST-967 bytes plus exactly the GST-508+GST-464 lines (the merged file minus main equals the wave minus fc292e22f, line for line), so GST-967's dispatch-superset no-loss check and GST-508's no-loss replay both hold; merged-head checks in GStack Minder drive handoffs/GSTA-202/merge-6841-checks.md"} diff --git a/test/fixtures/review-pins/replays/GST-508+GST-464.json b/test/fixtures/review-pins/replays/GST-508+GST-464.json new file mode 100644 index 000000000..9f015c90b --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508+GST-464.json @@ -0,0 +1 @@ +{"replay":"GST-508+GST-464","justification":"wave review-53cfb5bfe011 merge of #3744 into main: ship/sections/review-army.md and adversarial.md carry the GST-508 no-loss-replayed exclude pathspec placeholders inside the brief bodies (handoffs/GSTA-202/noloss-results.md) and the GST-464 lead-instruction edits outside the cut spans, which its brief-identity no-loss check admitted"} diff --git a/test/fixtures/review-pins/replays/GST-508+R51+GST-911+GST-967.json b/test/fixtures/review-pins/replays/GST-508+R51+GST-911+GST-967.json new file mode 100644 index 000000000..951fd3780 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508+R51+GST-911+GST-967.json @@ -0,0 +1 @@ +{"replay":"GST-508+R51+GST-911+GST-967","justification":"wave review-53cfb5bfe011 merge of main 68410828f into #3852: lib/review-depth.ts is main's GST-967 bytes plus exactly the GST-508+R51+GST-911 lines (the merged file minus main equals the wave minus fc292e22f, line for line); GST-967's dispatch-superset no-loss check (test/review-depth.test.ts) passes on the merged file, and the GST-508+R51+GST-911 checks are rerun at the merged head (GStack Minder drive handoffs/GSTA-202/merge-6841-checks.md)"} diff --git a/test/fixtures/review-pins/replays/GST-508+R51+GST-911.json b/test/fixtures/review-pins/replays/GST-508+R51+GST-911.json new file mode 100644 index 000000000..215ee2208 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508+R51+GST-911.json @@ -0,0 +1 @@ +{"replay":"GST-508+R51+GST-911","justification":"wave review-53cfb5bfe011 merge of main 3d99c2249 (#3848) into #3852: lib/review-depth.ts keeps GST-508's review_exclude_paths (no-loss replay, GStack Minder drive handoffs/GSTA-202/noloss-results.md), R51 with GST-850's dispatch-superset no-loss check, and GST-911's SVG drawings with GST-911-redos (no-loss replay: 0 of 100 main commits changed, reports/GSTA-249-911-decisions.tsv; test/review-planner-misreads.test.ts, test/review-risk-redos.test.ts); lib/review-risk.ts is main's GST-911 bytes. Merged-head checks (handoffs/GSTA-202/merge-3d99c22-checks.md): Arm P on unit 3593 differs only in the exclude pathspec and the disclosure line, the same brief diff as before the merge, and GST-911 panel unit f5cb6ec1e plans, dispatches and renders byte-identical briefs under 3d99c2249 and the merged head"} diff --git a/test/fixtures/review-pins/replays/GST-508+R51.json b/test/fixtures/review-pins/replays/GST-508+R51.json new file mode 100644 index 000000000..a2150fc75 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508+R51.json @@ -0,0 +1 @@ +{"replay":"GST-508+R51","justification":"wave review-53cfb5bfe011 merge of #3744 into main: lib/review-depth.ts keeps both admitted changes, the GST-508 no-loss replay of review_exclude_paths (Arm V 33 of 41 blockers vs R 33 of 41, p_NI 0.0279; GStack Minder drive handoffs/GSTA-202/noloss-results.md) and the R51 risk-tier re-score with GST-850's dispatch-superset no-loss check: excluded paths sit beside the added and removed line maps, and the exclusion filter keeps the publishes-docs rule"} diff --git a/test/fixtures/review-pins/replays/GST-508.json b/test/fixtures/review-pins/replays/GST-508.json new file mode 100644 index 000000000..de89e9c89 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-508.json @@ -0,0 +1 @@ +{"replay":"GST-508","justification":"GST-508 (#3744, GSTA-202) preregistered no-loss replay of review_exclude_paths at ef2760f: Arm V found 33 of 41 blockers vs R 33 of 41, exact sign flip with delta 4, p_NI 0.0279 (pass); Arm P identical depth, dispatch and reader sets on 26 of 26 units (GStack Minder drive handoffs/GSTA-202/noloss-results.md)"} diff --git a/test/fixtures/review-pins/replays/GST-850.json b/test/fixtures/review-pins/replays/GST-850.json new file mode 100644 index 000000000..24a5c9a44 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-850.json @@ -0,0 +1 @@ +{"replay":"GST-850","justification":"GST-850 dispatch-superset no-loss check: on the 26 GSTA-167 no-loss units the branch plan (wider security and money classes, write-up text in the batched diff) drops no reviewer role main dispatches, so no saved blocker can be lost, and over the last 100 main commits no commit with a gate or risk class drops depth (test/review-risk-routing.test.ts, test/review-risk-linear.test.ts); reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-898.json b/test/fixtures/review-pins/replays/GST-898.json new file mode 100644 index 000000000..bc23e8b90 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-898.json @@ -0,0 +1 @@ +{"replay":"GST-898","justification":"GST-898 verdict-identity no-loss check: the linear rm rule, and the rmSync rule (which also raises security on any call with more than 500 characters before its closing parenthesis), give the same verdict as R51's on every changed line of the last 100 main commits (69,023 lines, a563f47 back 100 first-parent commits) and on 60,000 seeded lines per rule (test/review-risk-redos.test.ts), and no commit changes depth, class or risk class, so every diff dispatches main's reviewers unchanged; reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-911-redos.json b/test/fixtures/review-pins/replays/GST-911-redos.json new file mode 100644 index 000000000..e8d792596 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-911-redos.json @@ -0,0 +1 @@ +{"replay":"GST-911-redos","justification":"GST-911 ReDoS fix verdict-identity no-loss check (wave review-e7b551ebd865): SVG_REFUSED's url() entry, rewritten to run in linear time, refuses exactly the strings #3838's pattern refused (test/review-risk-redos.test.ts: 60,000 seeded strings and the hand-picked edges, and every SVG blob in the repository's history, with zero differences); SVG_MARKUP now refuses a DOCTYPE naming any document but W3C's SVG DTD (test/review-planner-misreads.test.ts), which only keeps more SVGs code and so drops no reviewer; GST-911's no-loss replay holds unchanged; reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-911.json b/test/fixtures/review-pins/replays/GST-911.json new file mode 100644 index 000000000..aefbfacb1 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-911.json @@ -0,0 +1 @@ +{"replay":"GST-911","justification":"GST-911 no-loss replay (GSTA-249/13): the planner over the last 100 first-parent main commits to f5cb6ec1e, origin/main's code against this branch, gate scan fixed at 0, changes no commit's depth, class, risk classes, targeted classes or SCOPE_AUTH; the three misreads (a drawing-only SVG, a session-named data file, a bare base) are proven on fixtures (test/review-planner-misreads.test.ts), where every SVG that can run script stays code with the security class; reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-940.json b/test/fixtures/review-pins/replays/GST-940.json new file mode 100644 index 000000000..b717cf0b5 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-940.json @@ -0,0 +1 @@ +{"replay":"GST-940","justification":"GST-940 verdict no-loss check: svgDrawing reads a style element's opening tag as quoted pairs with ASCII whitespace, walks its body over the raw text, refuses any & in it, reads comments as CSS and refuses a non-ASCII element or attribute name; on 62,805 unique SVG files (21,658 with a style element) it gives the same verdict as main at 4bbad2422 on every file, and the planner over the last 100 first-parent main commits to 4bbad2422 changes no commit's depth, class or risk classes; the only verdicts it changes are the GST-940 cases in test/review-planner-misreads.test.ts; rerun on the merge with main at 8f08cb76a (GST-967 merged), with the same results; rerun on the merge with main at 5f705f367 (#3852 and #3864 merged), with the same results"} diff --git a/test/fixtures/review-pins/replays/GST-942.json b/test/fixtures/review-pins/replays/GST-942.json new file mode 100644 index 000000000..5e606ca3d --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-942.json @@ -0,0 +1 @@ +{"replay":"GST-942","justification":"GST-942 dispatch-superset no-loss check: the rmSync rule no longer stops at the first ), a matching rm( rule is added, and both read a file's changed lines joined for a delete split across lines, so a recursive delete raises security on more diffs and never on fewer; over the last 100 first-parent main commits to 3d99c2249 (62,531 changed lines) the line rules match all 170 lines the old rule matched plus 19 more, and no commit changes depth, class or risk class; reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-967.json b/test/fixtures/review-pins/replays/GST-967.json new file mode 100644 index 000000000..8749b2164 --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-967.json @@ -0,0 +1 @@ +{"replay":"GST-967","justification":"GST-967 dispatch-superset no-loss check (test/review-depth.test.ts, 'GST-967 dispatch-superset no-loss check'): over 1,440 planReview cases the branch plans exactly what main planned at 1022f4f except the 71 gate-class stops before Step 11, which add only Step 11's adversarial, outside and targeted voices, so no reviewer main dispatches is dropped or re-efforted; the brief bodies lib/review-brief.ts cuts from the ship sections hash to main's; only the lead's repeat-decision and Step 11 instructions changed; reviewer text is unchanged"} diff --git a/test/fixtures/review-pins/replays/GST-986.json b/test/fixtures/review-pins/replays/GST-986.json new file mode 100644 index 000000000..09c28491e --- /dev/null +++ b/test/fixtures/review-pins/replays/GST-986.json @@ -0,0 +1 @@ +{"replay":"GST-986","justification":"GST-986 brief-identity no-loss check (the brief-body test below): every reviewer brief lib/review-brief.ts cuts from ship/sections/adversarial.md is byte-identical to the bodies at the GST-257 pin; only the lead's Step 11 instruction naming the targeted record's path changed, outside the cut spans"} diff --git a/test/fixtures/review-pins/replays/GSTA-224-H1.json b/test/fixtures/review-pins/replays/GSTA-224-H1.json new file mode 100644 index 000000000..5bf962098 --- /dev/null +++ b/test/fixtures/review-pins/replays/GSTA-224-H1.json @@ -0,0 +1 @@ +{"replay":"GSTA-224 H1","justification":"GSTA-224 H1 preregistered no-loss run: the PR-type checklist against main's checklist and a no-gstack arm (Opus 5.5, 81 calls), 5 behavior cases, 10 Round 7 application blockers and 2 clean controls; every bar passed (handoffs/GSTA-224/h1-noloss-results.md)"} diff --git a/test/fixtures/review-pins/replays/L.json b/test/fixtures/review-pins/replays/L.json new file mode 100644 index 000000000..565698246 --- /dev/null +++ b/test/fixtures/review-pins/replays/L.json @@ -0,0 +1 @@ +{"replay":"L","justification":"GSTA-167 variant G Arm L: live no-loss replay of one batched reader (testing and security plus scoped checklists) and the Red Team on its findings, 2 passes, scored L∘G1 against G1 on the paired rule (part 2)"} diff --git a/test/fixtures/review-pins/replays/PR-a.json b/test/fixtures/review-pins/replays/PR-a.json new file mode 100644 index 000000000..f0db3c389 --- /dev/null +++ b/test/fixtures/review-pins/replays/PR-a.json @@ -0,0 +1 @@ +{"replay":"PR-a","justification":"GSTA-167 variant G Arm 4: structural plan identity at the 14 cohort heads and the free exact re-score of saved C, D and A' findings (parts 4a and 4b)"} diff --git a/test/fixtures/review-pins/replays/R51.json b/test/fixtures/review-pins/replays/R51.json new file mode 100644 index 000000000..f95ea5275 --- /dev/null +++ b/test/fixtures/review-pins/replays/R51.json @@ -0,0 +1 @@ +{"replay":"R51","justification":"R51 risk-tier routing no-loss re-score (GSTA-167): the saved panel findings (349 records over 24 of the 26 no-loss units) re-scored against the branch's plan lose no blocker whose only finders are roles it drops, and every gate, security, user-data and money fixture keeps main's reviewers plus the full panel (test/review-targeted.test.ts, test/review-risk-routing.test.ts); reviewer text is unchanged"} diff --git a/test/harness-version.test.ts b/test/harness-version.test.ts index 9f675cd62..5cb23c9c4 100644 --- a/test/harness-version.test.ts +++ b/test/harness-version.test.ts @@ -8,7 +8,7 @@ */ import { describe, expect, test } from 'bun:test'; import * as path from 'node:path'; -import { decide, formatDrift, harnessDrift, hasDrift, MANIFEST, readManifest, serializeManifest, workingHarnessFiles, +import { decide, formatDrift, harnessDrift, hasDrift, MANIFEST, PINS_DIR, readManifest, workingHarnessFiles, writeManifest, type HarnessManifest } from '../scripts/bump-harness-version'; import { caseSeriesIdentitiesV2, globRegex, HARNESS_PATTERNS, HARNESS_VERSION, isCaseOwnedFile, skillDirs, stampTrialSeries, treeEntries, type TreeEntry } from '../scripts/eval-trial-series'; @@ -18,12 +18,14 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import { LIVE_REGISTRY, type Registry } from '../scripts/eval-flake-rank'; import { matchGlob } from './helpers/test-selection'; +import { mergePinEdits, type PinMerge } from './helpers/pin-merge-fixture'; const ROOT = path.resolve(import.meta.dir, '..'); describe('HARNESS_VERSION guard', () => { test('every pinned shared-harness file matches scripts/harness-version.json', () => { expect(MANIFEST).toBe('scripts/harness-version.json'); + expect(PINS_DIR).toBe('scripts/harness-pins'); const manifest = readManifest(ROOT); expect(manifest.version).toBe(HARNESS_VERSION); expect(manifest.patterns).toEqual([...HARNESS_PATTERNS]); @@ -54,10 +56,107 @@ describe('HARNESS_VERSION guard', () => { expect(() => decide(base, now, 'bump', 'short')).toThrow('reason'); }); - test('the manifest serializes one entry per line, sorted, as valid JSON', () => { - const text = serializeManifest({ ...base, files: { 'z.ts': { blob: 'z', since: 1 }, 'a.ts': { blob: 'a', since: 1 } } }); - expect(JSON.parse(text).files).toEqual({ 'a.ts': { blob: 'a', since: 1 }, 'z.ts': { blob: 'z', since: 1 } }); - expect(text.split('\n').filter(line => line.includes('"blob"'))).toEqual([' "a.ts": {"blob":"a","since":1},', ' "z.ts": {"blob":"z","since":1}']); + const copyManifest = (into: string) => { + fs.cpSync(path.join(ROOT, MANIFEST), path.join(into, MANIFEST)); + fs.cpSync(path.join(ROOT, PINS_DIR), path.join(into, PINS_DIR), { recursive: true }); + }; + + test('the manifest writes one pin file per harness file under scripts/harness-pins and reads back unchanged (GST-1084)', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-pins-')); + try { + copyManifest(dir); + expect(readManifest(dir)).toEqual(readManifest(ROOT)); + const next: HarnessManifest = { version: 3, patterns: ['test/helpers/x/**'], files: { 'z.ts': { blob: 'z', since: 1 }, 'test/a.ts': { blob: 'a', since: 1, nonBehavioral: 'why' } } }; + writeManifest(dir, next); + expect(readManifest(dir)).toEqual(next); + expect(fs.readFileSync(path.join(dir, MANIFEST), 'utf8')).toBe('{\n "version": 3,\n "patterns": [\n "test/helpers/x/**"\n ]\n}\n'); + expect(fs.readFileSync(path.join(dir, PINS_DIR, 'test/a.ts.json'), 'utf8')).toBe('{"blob":"a","since":1,"nonBehavioral":"why"}\n'); + expect(fs.existsSync(path.join(dir, PINS_DIR, 'test/helpers/eval-store.ts.json'))).toBe(false); + fs.writeFileSync(path.join(dir, PINS_DIR, 'z.ts.txt'), 'stray\n'); + expect(() => readManifest(dir)).toThrow('every harness pin is .json'); + fs.rmSync(path.join(dir, PINS_DIR, 'z.ts.txt')); + fs.writeFileSync(path.join(dir, PINS_DIR, 'z.ts.json'), '<<<<<<< ours\n'); + expect(() => readManifest(dir)).toThrow(`${PINS_DIR}/z.ts.json: `); + fs.writeFileSync(path.join(dir, PINS_DIR, 'z.ts.json'), '{"blob":"z","since":1}\n'); + fs.writeFileSync(path.join(dir, MANIFEST), JSON.stringify({ version: 3, patterns: [], files: {} })); + expect(() => readManifest(dir)).toThrow('holds only version and patterns'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + // Value: protects=the one-file-per-pin layout still fails on every stale pin; fails_when=a pin with an old blob, a deleted pin + // or a pin for a file that is gone passes the guard; seam=readManifest and harnessDrift on the checkout's harness files + test('a stale, missing or orphaned harness pin fails the guard (GST-1084)', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-stale-')); + try { + copyManifest(dir); + const live = readManifest(ROOT); + const current = workingHarnessFiles(ROOT, live.patterns); + const pin = (file: string) => path.join(dir, PINS_DIR, `${file}.json`); + fs.writeFileSync(pin('test/helpers/eval-store.ts'), `${JSON.stringify({ ...live.files['test/helpers/eval-store.ts'], blob: '0'.repeat(40) })}\n`); + fs.rmSync(pin('test/helpers/e2e-gate.ts')); + fs.writeFileSync(pin('test/helpers/gone.ts'), `${JSON.stringify({ blob: '1'.repeat(40), since: live.version })}\n`); + const drift = harnessDrift(readManifest(dir), current); + expect(drift).toEqual({ changed: ['test/helpers/eval-store.ts'], added: ['test/helpers/e2e-gate.ts'], removed: ['test/helpers/gone.ts'] }); + expect(formatDrift(drift, live.version)).toContain(`Merge conflict in ${MANIFEST} or ${PINS_DIR}/: take the higher version, then rerun the script.`); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + // Value: protects=a --bump that fails partway (pins written, manifest version not) cannot pass the guard with behavior pooled into the + // old series; fails_when=a pin ahead of the manifest version reads as no drift; seam=writeManifest failing on a real write, then harnessDrift + test('a bump that fails partway leaves a pin ahead of the manifest version, and the guard reports it (GST-1084)', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-partial-')); + try { + copyManifest(dir); + const live = readManifest(dir); + const current = workingHarnessFiles(ROOT, live.patterns); + const [first, second] = current.map(entry => entry.file); + const blocked = path.join(dir, PINS_DIR, `${second}.json`); + const original = fs.readFileSync(blocked, 'utf8'); + fs.rmSync(blocked); + fs.mkdirSync(blocked); + expect(() => writeManifest(dir, decide(live, current, 'bump', 'behavior changed in the runner'))).toThrow(); + fs.rmdirSync(blocked); + fs.writeFileSync(blocked, original); + const after = readManifest(dir); + expect(after.version).toBe(live.version); + expect(after.files[first!]!.since).toBe(live.version + 1); + expect(harnessDrift(after, current).changed).toEqual([first]); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + // Value: protects=PRs that record decisions for different harness files merge without a conflict (the one-line-per-file manifest + // conflicted on adjacent lines and on the trailing comma); fails_when=two non-behavioral decisions on adjacent pins conflict in a + // real merge, or two decisions on one pin merge silently; seam=writeManifest and git merge-tree --write-tree on two branches + test('non-behavioral decisions on two different harness files merge cleanly in a real git merge (GST-1084)', () => { + const live = readManifest(ROOT); + const record = (file: string, blob: string, reason: string) => (root: string) => { + const manifest = readManifest(root); + const files = workingHarnessFiles(ROOT, live.patterns).map(entry => entry.file === file ? { file, blob } : entry); + writeManifest(root, decide(manifest, files, 'non-behavioral', reason)); + }; + const floor = 'test/helpers/pty/runners/floor.ts', observation = 'test/helpers/pty/runners/observation.ts'; + const merges: PinMerge[] = []; + try { + const clean = mergePinEdits(copyManifest, record(floor, 'a'.repeat(40), 'floor runner logging only'), record(observation, 'b'.repeat(40), 'observation runner comment only')); + merges.push(clean); + const conflict = mergePinEdits(copyManifest, record(floor, 'a'.repeat(40), 'floor runner logging only'), record(floor, 'c'.repeat(40), 'floor runner rename only')); + merges.push(conflict); + expect(clean.output).not.toContain('CONFLICT'); + expect(clean.clean).toBe(true); + expect(JSON.parse(clean.read(`${PINS_DIR}/${floor}.json`))).toEqual({ blob: 'a'.repeat(40), since: live.version, nonBehavioral: 'floor runner logging only' }); + expect(JSON.parse(clean.read(`${PINS_DIR}/${observation}.json`))).toEqual({ blob: 'b'.repeat(40), since: live.version, nonBehavioral: 'observation runner comment only' }); + expect(clean.read(MANIFEST)).toBe(fs.readFileSync(path.join(ROOT, MANIFEST), 'utf8')); + expect(conflict.clean).toBe(false); + expect(conflict.output).toContain(`CONFLICT (content): Merge conflict in ${PINS_DIR}/${floor}.json`); + } finally { + for (const merge of merges) fs.rmSync(merge.root, { recursive: true, force: true }); + } }); }); diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 1cc026c28..3b9f01e6c 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -209,6 +209,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/remote-identity-vectors.json': ['test/bin-context-windows-slug.test.ts', 'test/gstack-slug-parity.test.ts', 'test/remote-identity-vectors.test.ts'], 'test/fixtures/review-depth/**': ['test/review-dispatch-compare.test.ts'], 'test/fixtures/review-ledger/**': ['test/review-dispatch.test.ts'], + 'test/fixtures/review-pins/**': ['test/review-packets-removed.test.ts'], 'test/fixtures/review-risk/gbrain-evals-289.json': ['test/review-risk-routing.test.ts'], 'test/fixtures/review-design-lite-reports-ci-36633323521.json': ['test/review-finalization-budget.test.ts'], 'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'], diff --git a/test/helpers/pin-merge-fixture.ts b/test/helpers/pin-merge-fixture.ts new file mode 100644 index 000000000..8189b7280 --- /dev/null +++ b/test/helpers/pin-merge-fixture.ts @@ -0,0 +1,41 @@ +/** + * Two independent edits to a pin store, merged for real (GST-1084). seed(root) writes the base tree, edits a and b + * each change it on their own branch, and `git merge-tree --write-tree` says whether the two merge without a + * textual conflict. The repository is fresh under os.tmpdir(), outside any checkout, and git runs under gitEnv() so + * no global hook or config reaches it; the caller removes root. + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { gitEnv } from './git-env'; + +export interface PinMerge { root: string; clean: boolean; output: string; read: (rel: string) => string } + +export function mergePinEdits(seed: (root: string) => void, a: (root: string) => void, b: (root: string) => void): PinMerge { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pin-merge-')); + const git = (...args: string[]) => spawnSync('git', ['-c', 'user.name=Pin fixture', '-c', 'user.email=pins@gstack.test', '-c', 'commit.gpgsign=false', + '-c', 'core.autocrlf=false', ...args], { cwd: root, encoding: 'utf8', timeout: 30_000, env: gitEnv() }); + const must = (...args: string[]) => { + const result = git(...args); + if (result.status !== 0) throw new Error(`git ${args.join(' ')} failed (${result.status}): ${result.stderr}`); + return result.stdout; + }; + const commit = (edit: (root: string) => void, message: string) => { edit(root); must('add', '-A'); must('commit', '-q', '--no-verify', '-m', message); }; + let merged: ReturnType; + try { + must('init', '-q', '-b', 'base'); + commit(seed, 'seed'); + for (const [branch, edit] of [['a', a], ['b', b]] as const) { + must('checkout', '-q', '-b', branch, 'base'); + commit(edit, branch); + } + merged = git('merge-tree', '--write-tree', 'a', 'b'); + if (merged.status !== 0 && merged.status !== 1) throw new Error(`git merge-tree failed (${merged.status}): ${merged.stderr}`); + } catch (e) { + fs.rmSync(root, { recursive: true, force: true }); + throw e; + } + const tree = merged.stdout.split('\n')[0]!; + return { root, clean: merged.status === 0, output: merged.stdout, read: rel => must('show', `${tree}:${rel}`) }; +} diff --git a/test/review-depth.test.ts b/test/review-depth.test.ts index 4720ba610..befbe0e2f 100644 --- a/test/review-depth.test.ts +++ b/test/review-depth.test.ts @@ -149,7 +149,7 @@ describe('GST-967 invariant: no gate-class owner exit before Step 11\'s voices; }); /** - * The GST-967 dispatch-superset no-loss check (crew rule 2026-10-30; replay 'GST-967' in test/review-packets-removed.test.ts). + * The GST-967 dispatch-superset no-loss check (crew rule 2026-10-30; replay 'GST-967' in test/fixtures/review-pins/replays/GST-967.json). * On a 1,440-case grid (five diffs: light, standard, gate, docs, tests; passes 0-2; last none, full or fix-check; every mix of * the four fix flags; Step 11 run or not), main's planReview at 1022f4f (v1.92.81.0) hashes to MAIN_PLANS. The branch's plan, * with each voices step read back as main's stop (its reason is the parenthesised tail, no dispatch), hashes to the same value, diff --git a/test/review-packets-removed.test.ts b/test/review-packets-removed.test.ts index c1cf4fc10..2c520f690 100644 --- a/test/review-packets-removed.test.ts +++ b/test/review-packets-removed.test.ts @@ -11,6 +11,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { sectionTexts } from '../lib/review-brief'; +import { mergePinEdits, type PinMerge } from './helpers/pin-merge-fixture'; const ROOT = path.join(import.meta.dir, '..'); const REMOVED = [ @@ -49,71 +50,88 @@ describe('reviewer packets stay reverted (GSTA-167)', () => { * reviewer sees or how its findings rate, so it merges only after its preregistered no-loss replay * passes (GStack Minder crew rule, 2026-10-30); a cost replay alone is not enough (#3621 passed its * cost replay, then its no-loss replay found 11 of 30 blockers). Every pin names the replay that - * admits its bytes: repin with that replay's id here and in the commit message, and merge only on + * admits its bytes: repin with that replay's id in the pin file and in the commit message, and merge only on * its passing result. + * + * The pins live one file per pin under test/fixtures/review-pins (GST-1084), so two PRs that repin + * different files, or add different replays, never edit the same lines and merge without a conflict: + * files/.json {"sha", "replay"}: the sha256 of the file's LF-normalized bytes and the replay that admits them + * replays/.json {"replay", "justification"}: one replay id and its written no-loss justification, + * where is replaySlug(replay) + * brief-bodies/.json {"sha"}: one brief body lib/review-brief.ts cuts from the ship sections (below) */ -const NO_LOSS_REPLAYS = { - '#3638': 'no-loss round 3: main restored the pre-#3621 reviewers (v1.91.331.0)', - 'PR-a': "GSTA-167 variant G Arm 4: structural plan identity at the 14 cohort heads and the free exact re-score of saved C, D and A' findings (parts 4a and 4b)", - 'G1': "GSTA-167 variant G Arm G1: live no-loss replay of both prose voices with part 1's prompts, 2 passes, normalizer v2, paired rule against pooled C' (parts 1 and 3; specialists at medium as in C and C2)", - 'L': "GSTA-167 variant G Arm L: live no-loss replay of one batched reader (testing and security plus scoped checklists) and the Red Team on its findings, 2 passes, scored L∘G1 against G1 on the paired rule (part 2)", - 'GST-508': "GST-508 (#3744, GSTA-202) preregistered no-loss replay of review_exclude_paths at ef2760f: Arm V found 33 of 41 blockers vs R 33 of 41, exact sign flip with delta 4, p_NI 0.0279 (pass); Arm P identical depth, dispatch and reader sets on 26 of 26 units (GStack Minder drive handoffs/GSTA-202/noloss-results.md)", - 'GST-257': "GST-257 PR 1 dispatch-superset no-loss check (test/review-targeted.test.ts): on every fixture diff the branch dispatches main's reviewers at 8be7f21 unchanged plus only the targeted reader, and over the last 100 non-release commits on main it lost none; adds the targeted role and brief to the G1 and L bytes, whose existing reviewer text is unchanged", - 'GST-464': "GST-464 brief-identity no-loss check (the brief-body test below): every reviewer brief lib/review-brief.ts cuts from ship/sections/review-army.md and adversarial.md is byte-identical to the bodies at the L and GST-257 pins; only lead instructions outside the cut spans changed", - 'GST-850': "GST-850 dispatch-superset no-loss check: on the 26 GSTA-167 no-loss units the branch plan (wider security and money classes, write-up text in the batched diff) drops no reviewer role main dispatches, so no saved blocker can be lost, and over the last 100 main commits no commit with a gate or risk class drops depth (test/review-risk-routing.test.ts, test/review-risk-linear.test.ts); reviewer text is unchanged", - 'GSTA-224 H1': 'GSTA-224 H1 preregistered no-loss run: the PR-type checklist against main\'s checklist and a no-gstack arm (Opus 5.5, 81 calls), 5 behavior cases, 10 Round 7 application blockers and 2 clean controls; every bar passed (handoffs/GSTA-224/h1-noloss-results.md)', - 'R51': "R51 risk-tier routing no-loss re-score (GSTA-167): the saved panel findings (349 records over 24 of the 26 no-loss units) re-scored against the branch's plan lose no blocker whose only finders are roles it drops, and every gate, security, user-data and money fixture keeps main's reviewers plus the full panel (test/review-targeted.test.ts, test/review-risk-routing.test.ts); reviewer text is unchanged", - 'GST-508+R51': "wave review-53cfb5bfe011 merge of #3744 into main: lib/review-depth.ts keeps both admitted changes, the GST-508 no-loss replay of review_exclude_paths (Arm V 33 of 41 blockers vs R 33 of 41, p_NI 0.0279; GStack Minder drive handoffs/GSTA-202/noloss-results.md) and the R51 risk-tier re-score with GST-850's dispatch-superset no-loss check: excluded paths sit beside the added and removed line maps, and the exclusion filter keeps the publishes-docs rule", - 'GST-508+GST-464': "wave review-53cfb5bfe011 merge of #3744 into main: ship/sections/review-army.md and adversarial.md carry the GST-508 no-loss-replayed exclude pathspec placeholders inside the brief bodies (handoffs/GSTA-202/noloss-results.md) and the GST-464 lead-instruction edits outside the cut spans, which its brief-identity no-loss check admitted", - 'GST-986': "GST-986 brief-identity no-loss check (the brief-body test below): every reviewer brief lib/review-brief.ts cuts from ship/sections/adversarial.md is byte-identical to the bodies at the GST-257 pin; only the lead's Step 11 instruction naming the targeted record's path changed, outside the cut spans", - 'GST-942': "GST-942 dispatch-superset no-loss check: the rmSync rule no longer stops at the first ), a matching rm( rule is added, and both read a file's changed lines joined for a delete split across lines, so a recursive delete raises security on more diffs and never on fewer; over the last 100 first-parent main commits to 3d99c2249 (62,531 changed lines) the line rules match all 170 lines the old rule matched plus 19 more, and no commit changes depth, class or risk class; reviewer text is unchanged", - 'GST-911-redos': "GST-911 ReDoS fix verdict-identity no-loss check (wave review-e7b551ebd865): SVG_REFUSED's url() entry, rewritten to run in linear time, refuses exactly the strings #3838's pattern refused (test/review-risk-redos.test.ts: 60,000 seeded strings and the hand-picked edges, and every SVG blob in the repository's history, with zero differences); SVG_MARKUP now refuses a DOCTYPE naming any document but W3C's SVG DTD (test/review-planner-misreads.test.ts), which only keeps more SVGs code and so drops no reviewer; GST-911's no-loss replay holds unchanged; reviewer text is unchanged", - 'GST-911': "GST-911 no-loss replay (GSTA-249/13): the planner over the last 100 first-parent main commits to f5cb6ec1e, origin/main's code against this branch, gate scan fixed at 0, changes no commit's depth, class, risk classes, targeted classes or SCOPE_AUTH; the three misreads (a drawing-only SVG, a session-named data file, a bare base) are proven on fixtures (test/review-planner-misreads.test.ts), where every SVG that can run script stays code with the security class; reviewer text is unchanged", - 'GST-508+R51+GST-911': "wave review-53cfb5bfe011 merge of main 3d99c2249 (#3848) into #3852: lib/review-depth.ts keeps GST-508's review_exclude_paths (no-loss replay, GStack Minder drive handoffs/GSTA-202/noloss-results.md), R51 with GST-850's dispatch-superset no-loss check, and GST-911's SVG drawings with GST-911-redos (no-loss replay: 0 of 100 main commits changed, reports/GSTA-249-911-decisions.tsv; test/review-planner-misreads.test.ts, test/review-risk-redos.test.ts); lib/review-risk.ts is main's GST-911 bytes. Merged-head checks (handoffs/GSTA-202/merge-3d99c22-checks.md): Arm P on unit 3593 differs only in the exclude pathspec and the disclosure line, the same brief diff as before the merge, and GST-911 panel unit f5cb6ec1e plans, dispatches and renders byte-identical briefs under 3d99c2249 and the merged head", - 'GST-898': "GST-898 verdict-identity no-loss check: the linear rm rule, and the rmSync rule (which also raises security on any call with more than 500 characters before its closing parenthesis), give the same verdict as R51's on every changed line of the last 100 main commits (69,023 lines, a563f47 back 100 first-parent commits) and on 60,000 seeded lines per rule (test/review-risk-redos.test.ts), and no commit changes depth, class or risk class, so every diff dispatches main's reviewers unchanged; reviewer text is unchanged", - 'GST-967': "GST-967 dispatch-superset no-loss check (test/review-depth.test.ts, 'GST-967 dispatch-superset no-loss check'): over 1,440 planReview cases the branch plans exactly what main planned at 1022f4f except the 71 gate-class stops before Step 11, which add only Step 11's adversarial, outside and targeted voices, so no reviewer main dispatches is dropped or re-efforted; the brief bodies lib/review-brief.ts cuts from the ship sections hash to main's; only the lead's repeat-decision and Step 11 instructions changed; reviewer text is unchanged", - 'GST-508+GST-464+GST-967': "wave review-53cfb5bfe011 merge of main 68410828f into #3852: ship/sections/review-army.md and adversarial.md are main's GST-967 bytes plus exactly the GST-508+GST-464 lines (the merged file minus main equals the wave minus fc292e22f, line for line), so GST-967's dispatch-superset no-loss check and GST-508's no-loss replay both hold; merged-head checks in GStack Minder drive handoffs/GSTA-202/merge-6841-checks.md", - 'GST-508+R51+GST-911+GST-967': "wave review-53cfb5bfe011 merge of main 68410828f into #3852: lib/review-depth.ts is main's GST-967 bytes plus exactly the GST-508+R51+GST-911 lines (the merged file minus main equals the wave minus fc292e22f, line for line); GST-967's dispatch-superset no-loss check (test/review-depth.test.ts) passes on the merged file, and the GST-508+R51+GST-911 checks are rerun at the merged head (GStack Minder drive handoffs/GSTA-202/merge-6841-checks.md)", - 'GST-940': "GST-940 verdict no-loss check: svgDrawing reads a style element's opening tag as quoted pairs with ASCII whitespace, walks its body over the raw text, refuses any & in it, reads comments as CSS and refuses a non-ASCII element or attribute name; on 62,805 unique SVG files (21,658 with a style element) it gives the same verdict as main at 4bbad2422 on every file, and the planner over the last 100 first-parent main commits to 4bbad2422 changes no commit's depth, class or risk classes; the only verdicts it changes are the GST-940 cases in test/review-planner-misreads.test.ts; rerun on the merge with main at 8f08cb76a (GST-967 merged), with the same results; rerun on the merge with main at 5f705f367 (#3852 and #3864 merged), with the same results", -} as const; -type Replay = keyof typeof NO_LOSS_REPLAYS; -const REVIEWER_BRIEFS: Record = { - 'ship/sections/review-army.md': { sha: 'de3e0d35f2af56492c42fd287b6868fbe19bce96b86321e74a4947ba909b74eb', replay: 'GST-508+GST-464+GST-967' }, - 'ship/sections/adversarial.md': { sha: '0142d6bc3f2a5bceb9f8b2adef1d3a206e5836c99a0667fe995568da2e80a5a6', replay: 'GST-508+GST-464+GST-967' }, - 'review/sections/review-army.md': { sha: '78329b64640260cd8831ca81a9189f3f8dcade9c406431631d3f2cd798cd3967', replay: '#3638' }, - 'review/sections/adversarial.md': { sha: 'e6962fa25b78f362654f2954f79683e524d2ebd452d1494f8a2b099888e8e255', replay: 'G1' }, - 'review/checklist.md': { sha: 'a32963ef38d9d3613c58c805e8517cd2e593c63769b30dd7d40e2592fa48fc78', replay: 'GSTA-224 H1' }, - 'review/design-checklist.md': { sha: 'b9205f88313cc64902c0b13e9685722ab8138d3dcc58916154f47274abda7d83', replay: '#3638' }, - 'review/gate-integrity.md': { sha: 'f570db344c9a01df633bcaa281d6017f4f12420fd3e0aa3078c1ce06e0df8332', replay: '#3638' }, - 'review/specialists/api-contract.md': { sha: '263d23ac119dd601d315c191dbdbd503d47c00264c9c0ca81959559ca11d4e95', replay: '#3638' }, - 'review/specialists/data-migration.md': { sha: 'b6fd9eb229002ea598f8fe9ff53b1cd8821e3bd37a7aa7b07b5526556c71ebca', replay: '#3638' }, - 'review/specialists/maintainability.md': { sha: 'f8015d549e70297f782f0aabf6fe3a9fe31ec5ed61c311ab854e4a858282ed9b', replay: '#3638' }, - 'review/specialists/performance.md': { sha: '545c294ae53638b4c8524e8cde08246a4ce3b5c287ec7c44e27f5556a3b0e8cc', replay: '#3638' }, - 'review/specialists/red-team.md': { sha: '9ea05149f5b13d6a19ecec26285142d09d259cf80a515af828abdd1f43320427', replay: '#3638' }, - 'review/specialists/security.md': { sha: '17bfe813eb3de22835af048ec359770720f1d7f371a26460169f7e32b1bcac9f', replay: '#3638' }, - 'review/specialists/simplification.md': { sha: 'f33846b1dcf2fdeef82d0744ab2ef48fb4a33b34a737dafc33dd0e891756689f', replay: '#3638' }, - 'review/specialists/state-machines.md': { sha: '597215e4438d85d9cc71b48097c61eef7cba4f4b09dc14c92b4b94893b3d3ea2', replay: '#3638' }, - 'review/specialists/testing.md': { sha: 'd6cc2bef7be562387d0c0a22fefe646dff9c295f1d34d8453a6cb496584150e4', replay: '#3638' }, - 'review/specialists/targeted.md': { sha: '7f92a0d0a8319c3c0e076fb3b1a4689c45120b2be2c14dfc7d4cb03aafa9d6a5', replay: 'GST-257' }, - 'scripts/resolvers/fix-check.ts': { sha: '712bf5b0c439e96bac32f19cbe25394dabb4c94aa66600100bfde9b100e15a08', replay: '#3638' }, - 'lib/review-depth.ts': { sha: '25050a7a61f81df649d1df0d0237dbf961671356395317c6a527741f5bb56877', replay: 'GST-940' }, - 'lib/review-risk.ts': { sha: '08464b78a723ee463ca3a28d0abbc4eb8f4f512192061c703743707b66b5a311', replay: 'GST-942' }, -}; +const PINS = path.join(ROOT, 'test/fixtures/review-pins'); +const replaySlug = (replay: string) => replay.replace(/[^A-Za-z0-9+-]+/g, '-').replace(/^-+|-+$/g, ''); -/** - * The reviewer brief bodies lib/review-brief.ts cuts out of the ship sections, at the bytes replays L and GST-257 - * admitted, except fixtures, redTeam and native, which carry the exclude pathspec placeholders the GST-508 replay - * admitted (byte-identical to #3744 at 995e442). A section repin under GST-464 or GST-986 is valid only while every - * body still hashes here: the section may change outside the cut spans, never inside them. - */ -const BRIEF_BODIES: Record = { - specialist: '1ebe47ddb3291a9e4fe90b5a260eb57e328144959549bc95c119b68d23652cb5', - framing: 'aad667b5ea4c6d23d930dafffa38178c04be388cd4364fb73759fc92af17bb63', - fixtures: 'fbe52f3a9e0f4442eb755f1747a95944b373b439bd05c61f3eb68345d1061a5b', - redTeam: '6dd670ea040535ca6cceab81c29a395bb67ce5a2ab8fa2b81e51bc4711e4aada', - native: 'a7cc08fb31b6ca65932a316d589f0b09677426d193d63f88ec74fee6d93ce053', - outside: 'bd96406da28b03742384560425a333b6d6d7ed2be95bd2026c90e0039632cee2', -}; +/** Every pin file under dir, keyed by its path below dir without `.json`; any other file is an error. */ +function readPinDir(dir: string): Record { + const out: Record = {}; + for (const rel of (fs.readdirSync(dir, { recursive: true }) as string[]).map(f => f.split(path.sep).join('/')).sort()) { + if (fs.statSync(path.join(dir, rel)).isDirectory()) continue; + if (!rel.endsWith('.json')) throw new Error(`${dir}/${rel}: every pin file is .json`); + try { + out[rel.slice(0, -'.json'.length)] = JSON.parse(fs.readFileSync(path.join(dir, rel), 'utf8')); + } catch (e) { + throw new Error(`${dir}/${rel}: ${(e as Error).message}`); + } + } + return out; +} + +/** The pin store at pins, with every file that is not exactly its kind's fields as non-empty strings, or is misnamed. */ +function readPins(pins: string) { + const malformed: string[] = []; + const kind = (sub: string, keys: readonly K[]) => { + const out: Record> = {}; + for (const [name, value] of Object.entries(readPinDir(path.join(pins, sub)))) { + const fields = typeof value === 'object' && value !== null && !Array.isArray(value) ? value as Record : {}; + if (Object.keys(fields).sort().join() === [...keys].sort().join() && keys.every(k => typeof fields[k] === 'string' && fields[k] !== '')) out[name] = fields as Record; + else malformed.push(`${sub}/${name}.json (expected exactly ${keys.join(', ')})`); + } + return out; + }; + const replayFiles = kind('replays', ['replay', 'justification']); + for (const [name, r] of Object.entries(replayFiles)) if (name !== replaySlug(r.replay)) malformed.push(`replays/${name}.json (replay ${r.replay} belongs in replays/${replaySlug(r.replay)}.json)`); + return { + malformed, + replays: Object.fromEntries(Object.values(replayFiles).map(r => [r.replay, r.justification])), + files: kind('files', ['sha', 'replay']), + bodies: Object.fromEntries(Object.entries(kind('brief-bodies', ['sha'])).map(([name, body]) => [name, body.sha])), + }; +} +type Pins = ReturnType; + +/** Every source a reviewer brief, checklist or depth rule comes from: deleting one of these pins fails, never drops the check. */ +const MUST_PIN = [ + 'lib/review-depth.ts', + 'lib/review-risk.ts', + 'review/checklist.md', + 'review/design-checklist.md', + 'review/gate-integrity.md', + 'review/sections/adversarial.md', + 'review/sections/review-army.md', + 'scripts/resolvers/fix-check.ts', + 'ship/sections/adversarial.md', + 'ship/sections/review-army.md', +]; + +/** MUST_PIN entries and specialist checklists under root with no pin. */ +const unpinned = (pins: Pins, root: string) => [...MUST_PIN, ...fs.readdirSync(path.join(root, 'review/specialists')).filter(f => f.endsWith('.md')).map(f => `review/specialists/${f}`)] + .filter(f => !Object.hasOwn(pins.files, f)); + +/** Pinned files whose LF-normalized bytes under root no longer hash to their pin. */ +const stalePins = (pins: Pins, root: string) => Object.entries(pins.files).filter(([file, pin]) => { + const text = fs.readFileSync(path.join(root, file), 'utf8').replace(/\r\n/g, '\n'); + return createHash('sha256').update(text).digest('hex') !== pin.sha; +}).map(([file, pin]) => `${file} (a repin needs a passing no-loss replay; the current pin names ${pin.replay})`); + +/** Pins naming no replay in the store, and replays whose justification names no no-loss replay or re-score. */ +const unadmitted = (pins: Pins) => [ + ...Object.entries(pins.files).filter(([, pin]) => !Object.hasOwn(pins.replays, pin.replay)).map(([file, pin]) => `${file} names unknown replay ${pin.replay}`), + ...Object.entries(pins.replays).filter(([, what]) => !/no-loss|re-score/.test(what)).map(([replay]) => `replay ${replay} is not a no-loss replay or re-score`), +]; + +const PINNED = readPins(PINS); describe('reviewer briefs stay at their no-loss-tested bytes (GSTA-167)', () => { // Value: protects=the GST-986 repin of ship/sections/adversarial.md admits only a lead-instruction edit: every brief a reviewer @@ -121,34 +139,103 @@ describe('reviewer briefs stay at their no-loss-tested bytes (GSTA-167)', () => // section pin moves; seam=sectionTexts, the function every render uses; no_claim=the checklists (pinned by file hash below) test('every reviewer brief cut from the ship sections is byte-identical to its pinned body', () => { const hashes = (root: string) => Object.fromEntries(Object.entries(sectionTexts(root)).map(([k, v]) => [k, createHash('sha256').update(v).digest('hex')])); - expect(hashes(ROOT)).toEqual(BRIEF_BODIES); + expect(hashes(ROOT)).toEqual(PINNED.bodies); const copy = fs.mkdtempSync(path.join(os.tmpdir(), 'brief-bodies-')); try { fs.cpSync(path.join(ROOT, 'ship', 'sections'), path.join(copy, 'ship', 'sections'), { recursive: true }); const adversarial = path.join(copy, 'ship', 'sections', 'adversarial.md'); fs.writeFileSync(adversarial, fs.readFileSync(adversarial, 'utf8').replace('Recommendation:', 'Recommendation (short):')); - expect(hashes(copy)).not.toEqual(BRIEF_BODIES); + expect(hashes(copy)).not.toEqual(PINNED.bodies); } finally { fs.rmSync(copy, { recursive: true, force: true }); } }); test('every pinned reviewer brief, checklist and depth rule hashes to its pin', () => { - const drift = Object.entries(REVIEWER_BRIEFS).filter(([file, pin]) => { - const text = fs.readFileSync(path.join(ROOT, file), 'utf8').replace(/\r\n/g, '\n'); - return createHash('sha256').update(text).digest('hex') !== pin.sha; - }).map(([file]) => `${file} (a repin needs a passing no-loss replay; the current pin names ${REVIEWER_BRIEFS[file]!.replay})`); - expect(drift).toEqual([]); + expect(PINNED.malformed).toEqual([]); + expect(stalePins(PINNED, ROOT)).toEqual([]); }); test('every pin names the preregistered no-loss replay that admits its bytes', () => { - const unnamed = Object.entries(REVIEWER_BRIEFS).filter(([, pin]) => !(pin.replay in NO_LOSS_REPLAYS)).map(([file]) => file); - expect(unnamed).toEqual([]); - for (const what of Object.values(NO_LOSS_REPLAYS)) expect(what).toMatch(/no-loss|re-score/); + expect(unadmitted(PINNED)).toEqual([]); }); test('the pins cover every specialist checklist', () => { const specialists = fs.readdirSync(path.join(ROOT, 'review/specialists')).filter(f => f.endsWith('.md')).map(f => `review/specialists/${f}`); - expect(specialists.filter(f => !(f in REVIEWER_BRIEFS))).toEqual([]); + expect(specialists.filter(f => !Object.hasOwn(PINNED.files, f))).toEqual([]); + }); + + test('every source a reviewer brief, checklist or depth rule comes from has a pin (GST-1084)', () => { + expect(unpinned(PINNED, ROOT)).toEqual([]); + }); + + // Value: protects=the one-file-per-pin layout keeps every check the in-file table had; fails_when=a stale sha, an unknown or + // unjustified replay, a malformed or misnamed pin file, or a stray file passes; seam=readPins, stalePins and unadmitted, the + // functions the checks above call; no_claim=that a repin's replay actually passed (the commit and PR carry that) + test('a stale pin, an unknown or unjustified replay and a malformed pin file each fail (GST-1084)', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'review-pins-')); + try { + const src = path.join(tmp, 'src'), pins = path.join(tmp, 'pins'); + for (const file of Object.keys(PINNED.files)) fs.cpSync(path.join(ROOT, file), path.join(src, file)); + fs.cpSync(PINS, pins, { recursive: true }); + expect(stalePins(readPins(pins), src)).toEqual([]); + fs.writeFileSync(path.join(src, 'review/checklist.md'), fs.readFileSync(path.join(src, 'review/checklist.md'), 'utf8').replace(/\n/g, '\r\n')); + expect(stalePins(readPins(pins), src)).toEqual([]); + fs.appendFileSync(path.join(src, 'review/checklist.md'), 'One more rule.\r\n'); + expect(stalePins(readPins(pins), src)).toEqual([`review/checklist.md (a repin needs a passing no-loss replay; the current pin names ${PINNED.files['review/checklist.md']!.replay})`]); + + const pin = (file: string, value: unknown) => fs.writeFileSync(path.join(pins, 'files', `${file}.json`), `${JSON.stringify(value)}\n`); + pin('review/design-checklist.md', { sha: PINNED.files['review/design-checklist.md']!.sha, replay: 'GST-0000' }); + pin('lib/review-risk.ts', { sha: PINNED.files['lib/review-risk.ts']!.sha, replay: 'toString' }); + fs.writeFileSync(path.join(pins, 'replays', 'GST-0001.json'), `${JSON.stringify({ replay: 'GST-0001', justification: 'reviewed by hand' })}\n`); + expect(unadmitted(readPins(pins))).toEqual(['lib/review-risk.ts names unknown replay toString', 'review/design-checklist.md names unknown replay GST-0000', + 'replay GST-0001 is not a no-loss replay or re-score']); + + expect(unpinned(readPins(pins), ROOT)).toEqual([]); + fs.rmSync(path.join(pins, 'files', 'lib/review-risk.ts.json')); + fs.rmSync(path.join(pins, 'files', 'review/specialists/testing.md.json')); + expect(unpinned(readPins(pins), ROOT)).toEqual(['lib/review-risk.ts', 'review/specialists/testing.md']); + + pin('review/gate-integrity.md', { ...PINNED.files['review/gate-integrity.md'], note: 'extra' }); + fs.writeFileSync(path.join(pins, 'replays', 'GST-0002.json'), `${JSON.stringify({ replay: 'GST-0003', justification: 'a no-loss replay' })}\n`); + expect(readPins(pins).malformed).toEqual(['replays/GST-0002.json (replay GST-0003 belongs in replays/GST-0003.json)', + 'files/review/gate-integrity.md.json (expected exactly sha, replay)']); + fs.writeFileSync(path.join(pins, 'files', 'review/gate-integrity.md.json'), '<<<<<<< a\n{}\n'); + expect(() => readPins(pins)).toThrow('files/review/gate-integrity.md.json: '); + fs.rmSync(path.join(pins, 'files', 'review/gate-integrity.md.json')); + fs.writeFileSync(path.join(pins, 'brief-bodies', 'specialist.txt'), 'stray\n'); + expect(() => readPins(pins)).toThrow('every pin file is .json'); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + // Value: protects=PRs that repin different review files merge without a conflict (five main-merge conflicts on 2026-11-02 came + // from the in-file table); fails_when=two repins of adjacent pins, each with its own new replay, conflict in a real merge, or + // two repins of one pin merge silently; seam=git merge-tree --write-tree on two branches; no_claim=semantic merge correctness + test('repins of two different files, each with its own new replay, merge cleanly in a real git merge (GST-1084)', () => { + const repin = (file: string, sha: string, replay: string) => (root: string) => { + fs.writeFileSync(path.join(root, 'files', `${file}.json`), `${JSON.stringify({ sha, replay })}\n`); + fs.writeFileSync(path.join(root, 'replays', `${replaySlug(replay)}.json`), `${JSON.stringify({ replay, justification: `${replay} no-loss replay` })}\n`); + }; + const seed = (root: string) => fs.cpSync(PINS, root, { recursive: true }); + const army = 'ship/sections/review-army.md', adversarial = 'ship/sections/adversarial.md'; + const merges: PinMerge[] = []; + try { + const clean = mergePinEdits(seed, repin(army, 'a'.repeat(64), 'GST-9001+GST-967'), repin(adversarial, 'b'.repeat(64), 'GST-9002')); + merges.push(clean); + const conflict = mergePinEdits(seed, repin(army, 'a'.repeat(64), 'GST-9001'), repin(army, 'c'.repeat(64), 'GST-9003')); + merges.push(conflict); + expect(clean.output).not.toContain('CONFLICT'); + expect(clean.clean).toBe(true); + expect(JSON.parse(clean.read(`files/${army}.json`))).toEqual({ sha: 'a'.repeat(64), replay: 'GST-9001+GST-967' }); + expect(JSON.parse(clean.read(`files/${adversarial}.json`))).toEqual({ sha: 'b'.repeat(64), replay: 'GST-9002' }); + expect(JSON.parse(clean.read('replays/GST-9001+GST-967.json')).replay).toBe('GST-9001+GST-967'); + expect(JSON.parse(clean.read('replays/GST-9002.json')).replay).toBe('GST-9002'); + expect(conflict.clean).toBe(false); + expect(conflict.output).toContain(`CONFLICT (content): Merge conflict in files/${army}.json`); + } finally { + for (const merge of merges) fs.rmSync(merge.root, { recursive: true, force: true }); + } }); }); diff --git a/test/ship-release.test.ts b/test/ship-release.test.ts index 8b7dc72b2..49d324c79 100644 --- a/test/ship-release.test.ts +++ b/test/ship-release.test.ts @@ -309,25 +309,34 @@ describe('gstack-ship-release push: branch names', () => { }); describe('GST-1031: a non-behavioral harness re-pin after review is release metadata', () => { - // Value: protects=a scripts/harness-version.json edit after review that only records changed harness files as non-behavioral at the same + // Value: protects=a scripts/harness-pins/.json edit after review that only records changed harness files as non-behavioral at the same // version routes as metadata, so a clean round 2 is not sent back through 5-11.5, while a version bump, a pattern change or an entry // without its reason stays behavior; fails_when=the re-pin stops finish, or a bump or bare re-pin routes as metadata; // why_new=GSTA-150 GST-1027 (icm_01M6ERWWBZ3VWXZY07F2W5ZSFB); seam=route; no_claim=that the recorded reason is true - const pin = (version: number, entry: Record, patterns = ['test/helpers/x.ts']) => JSON.stringify({ version, patterns, files: { 'test/helpers/x.ts': entry } }) + '\n'; - const finish = (after: string) => { + const PIN = 'scripts/harness-pins/test/helpers/x.ts.json'; + const pin = (version: number, entry: Record, patterns = ['test/helpers/x.ts']) => ({ + 'scripts/harness-version.json': JSON.stringify({ version, patterns }) + '\n', + [PIN]: JSON.stringify(entry) + '\n', + }); + const finish = (after: Record) => { const f = fx(); - f.write({ 'scripts/harness-version.json': pin(6, { blob: 'a'.repeat(40), since: 6 }) }); f.commit('pin'); + f.write(pin(6, { blob: 'a'.repeat(40), since: 6 })); f.commit('pin'); const reviewed = wtree(f); const post = snapshot(f); - f.write({ 'scripts/harness-version.json': after }); f.commit('re-pin'); + f.write(after); f.commit('re-pin'); return f.run(RELEASE, ['finish', '--base', 'main', '--reviewed-tree', reviewed, '--docs-post', post, '--lane', LANE]).stdout; }; test('a non-behavioral re-pin is metadata; a bump, a pattern change or an entry without its reason is behavior', () => { const repin = finish(pin(6, { blob: 'b'.repeat(40), since: 6, nonBehavioral: 'comment-only edit: no verdict can change' })); expect(lineOf(repin, 'ROUTE')).toMatch(/behavior=0 docs=0 metadata=1/); expect(repin).not.toContain('STOP: behavior'); - for (const after of [pin(7, { blob: 'b'.repeat(40), since: 7 }), pin(6, { blob: 'b'.repeat(40), since: 6 }), pin(6, { blob: 'b'.repeat(40), since: 6, nonBehavioral: 'comment-only edit: no verdict can change' }, ['test/helpers/**'])]) - expect(finish(after)).toContain('STOP: behavior, tests or build inputs changed since review (scripts/harness-version.json)'); + expect(finish(pin(7, { blob: 'b'.repeat(40), since: 7 }))).toContain(`STOP: behavior, tests or build inputs changed since review (${PIN},scripts/harness-version.json)`); + expect(finish(pin(6, { blob: 'b'.repeat(40), since: 6 }))).toContain(`STOP: behavior, tests or build inputs changed since review (${PIN})`); + expect(finish(pin(6, { blob: 'b'.repeat(40), since: 6, nonBehavioral: 'comment-only edit: no verdict can change' }, ['test/helpers/**']))).toContain(`STOP: behavior, tests or build inputs changed since review (${PIN},scripts/harness-version.json)`); + }); + test('the manifest regaining a files table is behavior', () => { + const f = { 'scripts/harness-version.json': JSON.stringify({ version: 6, patterns: ['test/helpers/x.ts'], files: {} }) + '\n' }; + expect(finish(f)).toContain('STOP: behavior, tests or build inputs changed since review (scripts/harness-version.json)'); }); }); From 95fc2105ff24d114d5aa078c8a46f860a0e0a8d9 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 07:13:49 -0800 Subject: [PATCH 2/5] v1.94.21.0 fix(ship-receipt): a stacked carry ignores files the child's own earlier main merge brought in (GST-1214) (#3940) * fix(ship-receipt): a stacked carry ignores files the child's own earlier main merge brought in (GST-1214) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * changelog: GST-1214 fragment Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-receipt): never leave out a path with a compared path beneath it (review fix, GST-1214) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-ship-receipt | 18 +++++-- docs/ship-policy.md | 5 +- package.json | 2 +- test/ship-receipt-carry-squash.test.ts | 65 ++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c88c2bd..7250ce534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.94.21.0] - 2026-11-03 + +- **A stacked PR's /ship receipt now carries even when the child merged main before its receipt was written (GST-1214).** After the parent squash-landed and main was merged in again, `gstack-ship-receipt carry` compared the range from the parent's head, which still held every file the child's first main merge brought in, and refused with `RECEIPT_CARRY_PRODUCT_CHANGED` naming those files. The carry now leaves out paths whose content at the reviewed head is main's, unless HEAD changed them. A real edit to one of those files still refuses. + ## [1.94.20.0] - 2026-11-03 ### Changed diff --git a/VERSION b/VERSION index f9f39ad26..098e7bdd3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.20.0 +1.94.21.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 1c89466ae..96ec3a476 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.20.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.21.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-ship-receipt b/bin/gstack-ship-receipt index 15f1a15f3..e45094d57 100755 --- a/bin/gstack-ship-receipt +++ b/bin/gstack-ship-receipt @@ -666,11 +666,19 @@ function cmdCarry(rest: string[], json: boolean): number { // receipt written against the base while stacked reviewed the parent's hunks too, and the squash moved them into the // base (GST-1199). A parent counts only when merging it into the current base changes no product path. const measure = (rangeBase: string) => { - const listed = [touchedPaths(repoRoot, rangeBase, from), touchedPaths(repoRoot, headBase, head)]; - if (!listed[0] || !listed[1]) return `git could not list the paths the branch touched against ${base}; run /ship on ${head.slice(0, 12)}`; - const touched = new Set([...listed[0], ...listed[1]]); - const spec = productSpec(repoRoot, rules, from, head, { touched, sides: [[rangeBase, from], [headBase, head]], own: branchSides(repoRoot, baseRef, from, '') }); - if (!spec) return `git diff ${from.slice(0, 12)} ${head.slice(0, 12)} failed`; + const listed = [touchedPaths(repoRoot, rangeBase, from), touchedPaths(repoRoot, headBase, head), rangeBase === fromBase ? new Set() : touchedPaths(repoRoot, fromBase, from)]; + if (!listed[0] || !listed[1] || !listed[2]) return `git could not list the paths the branch touched against ${base}; run /ship on ${head.slice(0, 12)}`; + // From a stack parent, a path whose reviewed content is the base's (an earlier base merge brought it in) is base + // context, not reviewed product, unless HEAD changed it against the base (GST-1214). + // A pathspec also names everything beneath a path, so a path with any compared path under it (a file the base turned + // into a directory) stays in. + const compared = [...listed[0], ...listed[1]]; + const baseMerged = rangeBase === fromBase ? [] : [...listed[0]].filter(p => !listed[2]!.has(p) && !listed[1]!.has(p) && !compared.some(q => q.startsWith(`${p}/`))); + const touched = new Set([...listed[0], ...listed[1]].filter(p => !baseMerged.includes(p))); + const found = productSpec(repoRoot, rules, from, head, { touched, sides: [[rangeBase, from], [headBase, head]], own: branchSides(repoRoot, baseRef, from, '') }); + if (!found) return `git diff ${from.slice(0, 12)} ${head.slice(0, 12)} failed`; + const skip = baseMerged.map(p => `:(exclude,literal)${p}`); + const spec = { ...found, product: [...found.product, ...skip], tests: [...found.tests, ...skip] }; const ids = [patchId(repoRoot, rangeBase, from, spec.product), patchId(repoRoot, headBase, head, spec.product)]; for (const r of ids) if (typeof r !== 'string') return `git could not compute the product patch-id at one of the heads: ${r.error}`; return { rangeBase, touched, spec, productFrom: ids[0] as string, productHead: ids[1] as string }; diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 6612a8aa3..f611346eb 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1536,7 +1536,10 @@ the reviewed range (the reviewed head excluded, at most 100 commits) whose product change the current base already holds: `git merge-tree` of that commit into the base changes no product path. A conflicted product path never counts, so a parent that never landed is never treated as one, even when the child's -merge dropped its change. The comparison then runs from that commit: an equal +merge dropped its change. The comparison then runs from that commit, leaving out +every path whose content at the reviewed head is the base's (a base merge the +child made before its receipt brought it in) unless HEAD changed it against the +base (GST-1214): an equal patch-id carries with reason `stack parent landed`, base context next to the child's unchanged hunks is `RECEIPT_CARRY_SCOPED_REVIEW`, and any other difference refuses with `RECEIPT_CARRY_PRODUCT_CHANGED`. diff --git a/package.json b/package.json index 436bce3c2..eb743f2a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.20", + "version": "1.94.21", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/ship-receipt-carry-squash.test.ts b/test/ship-receipt-carry-squash.test.ts index a47e025c1..20db3a387 100644 --- a/test/ship-receipt-carry-squash.test.ts +++ b/test/ship-receipt-carry-squash.test.ts @@ -240,3 +240,68 @@ describe('carry after a stack parent squash-merges, receipt written against main expect(landedStackParent(repo.work, { rangeBase: fork, reviewed, currentBase: mainTip }, ['.', ':(exclude,glob)test/**'])).toBeNull(); }, 60_000); }); + +describe('carry after a stack parent lands, the child having merged main before its receipt (GST-1214)', () => { + /** main <- parent (lib/util.js) <- feature (lib/retry.js), which merges main (wiki/) once before its receipt is written against main. */ + function mergedStack(repo: FixtureRepo, mainFiles: Record = { 'wiki/a.md': '# a\n', 'wiki/b.md': '# b\n' }): { parent: string; reviewed: string; body: string } { + repo.git('checkout', '-q', '-b', 'parent'); + const parent = commit(repo, { 'lib/util.js': SUB }, 'parent: sub'); + repo.git('push', '-q', 'origin', 'parent'); + repo.git('checkout', '-q', 'feature'); + repo.git('reset', '-q', '--hard', parent); + commit(repo, { 'lib/retry.js': RETRY4 }, 'child: four retries'); + repo.git('checkout', '-q', 'main'); + commit(repo, mainFiles, 'main: wiki'); + repo.git('push', '-q', 'origin', 'main'); + repo.git('checkout', '-q', 'feature'); + repo.git('merge', '-q', '--no-edit', 'origin/main'); + const reviewed = repo.git('rev-parse', 'HEAD'); + const body = path.join(repo.root, 'body.md'); + const w = runBin(repo, 'gstack-ship-receipt', ['write', '--pr', '8', '--base', 'main', '--version', '1.0.0', '--session-kind', 'unattended', '--body', body]); + expect(w.status).toBe(EXIT.ok); + return { parent, reviewed, body }; + } + + // Value: protects=a stacked child that merged main before its receipt keeps it once the parent squash-lands and main is + // merged again (gstack-evals#178 refused on 20 wiki/ paths its first base merge brought in); fails_when=the range from the + // landed parent counts paths that hold main's content at the reviewed head; seam=none; no_claim=gh pr edit (--body stands in) + test('the child carries when only its earlier base merge separates the parent range from HEAD\'s diff against main', () => { + const repo = makeFixtureRepo(); repos.push(repo); + const { parent, reviewed, body } = mergedStack(repo); + landOnMain(repo, { 'lib/util.js': SUB }); + const head = repo.git('rev-parse', 'HEAD'); + const out = carry(repo, body); + expect(out.err).not.toContain('RECEIPT_CARRY_PRODUCT_CHANGED'); + expect(out.status).toBe(EXIT.ok); + expect(out.out).toContain(`CARRY: carried from ${reviewed.slice(0, 12)} to ${head.slice(0, 12)}`); + expect(receiptIn(body).carry!.reason).toBe(`stack parent ${parent.slice(0, 12)} landed + base merge`); + }, 90_000); + + // Value: protects=fail closed: an edit to a file the earlier base merge brought in is product and refuses; fails_when=every + // base-merged path drops out of both sides, the child's own edit included; seam=none; no_claim=- + test('an edit to a base-merged file after the receipt refuses with RECEIPT_CARRY_PRODUCT_CHANGED', () => { + const repo = makeFixtureRepo(); repos.push(repo); + const { reviewed, body } = mergedStack(repo); + landOnMain(repo, { 'lib/util.js': SUB }); + commit(repo, { 'wiki/a.md': '# a, edited by the child\n' }, 'child: edit wiki'); + const out = carry(repo, body); + expect(out.status).toBe(EXIT.refused); + expect(out.out).toContain('CARRY: refused (RECEIPT_CARRY_PRODUCT_CHANGED)'); + expect(out.err).toContain('wiki/a.md'); + expect(receiptIn(body).head).toBe(reviewed); + }, 90_000); + + // Value: protects=fail closed when the earlier base merge turned a file into a directory: leaving out the old file must not + // hide an edit beneath it (a pathspec names a path and everything under it); fails_when=the exclusion covers a changed + // descendant; seam=none; no_claim=- + test('a base merge that replaced a file with a directory never hides an edit beneath it', () => { + const repo = makeFixtureRepo({ baseFiles: { services: 'old\n' } }); repos.push(repo); + const { reviewed, body } = mergedStack(repo, { services: null, 'services/worker.js': 'exports.allowed = false;\n' }); + landOnMain(repo, { 'lib/util.js': SUB }); + commit(repo, { 'services/worker.js': 'exports.allowed = true;\n' }, 'child: allow'); + const out = carry(repo, body); + expect(out.status).toBe(EXIT.refused); + expect(out.out).toContain('CARRY: refused (RECEIPT_CARRY_PRODUCT_CHANGED)'); + expect(receiptIn(body).head).toBe(reviewed); + }, 90_000); +}); From 592fa0c8f354d6a96a6438fc4738451119792ed1 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 07:21:29 -0800 Subject: [PATCH 3/5] v1.94.22.0 feat(skills): when-to-use descriptions, wider triggers and contract blocks for /investigate, /review, /qa, /verify (GST-1088 lane B) (#3909) * feat(skills): when-to-use descriptions, wider triggers and contract blocks for /investigate, /review, /qa, /verify (GST-1088 lane B) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(skills): keep core trigger phrases in /review and /qa descriptions; bare gstack-skill-contract name until lane A ships it Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(skills): Done-means list leaves out steps a host suppresses and allows the small-diff skip (review fix round) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: Done-means honors loadout section exclusions and the skill's own stop rules (Step 11 voices) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: the Done-means checker sentence follows an explicit CONTRACT_CHECKER_SHIPPED flag Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: cite GST-1088 gate rev 2 for the /review cap raises Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Revert "test: cite GST-1088 gate rev 2 for the /review cap raises" This reverts commit cc1539cd97b004eb8d06680121f3a7d7c77672df. * fix: trim /qa's opening sentence by 4 bytes so the branch fits main's qa parity cap Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: flip CONTRACT_CHECKER_SHIPPED now that bin/gstack-skill-contract ships, and re-pin /review by the measured 240 bytes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: the changelog fragment says lane A's routing hook and contract check have landed Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- gstack/llms.txt | 8 +- investigate/SKILL.md | 33 ++- investigate/SKILL.md.tmpl | 32 ++- package.json | 2 +- qa/SKILL.md | 2 +- qa/SKILL.md.tmpl | 4 +- review/SKILL.md | 28 ++- review/SKILL.md.tmpl | 27 ++- scripts/resolvers/contract-done-means.ts | 142 +++++++++++++ scripts/resolvers/index.ts | 2 + test/fixtures/skill-md-size-allowlist.json | 18 +- test/harness-register.test.ts | 2 +- test/helpers/carve-guards.ts | 4 +- test/skill-descriptions.test.ts | 228 +++++++++++++++++++++ verify/SKILL.md | 4 +- verify/SKILL.md.tmpl | 10 +- 19 files changed, 525 insertions(+), 39 deletions(-) create mode 100644 scripts/resolvers/contract-done-means.ts create mode 100644 test/skill-descriptions.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7250ce534..e23f5c500 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.94.22.0] - 2026-11-03 + +**/investigate, /review, /qa and /verify now say when to use them, and /investigate and /review declare what "done" means (GST-1088 lane B).** Untold, Opus 5.5 never opened /investigate in 252 trials (Run D, gstack-evals#140). A when-to-use description raised GPT-6.1 Sol's open rate from 9.5% to 27.0% in the same run. Claude Code's catalog shows only a description's first sentence, so each of the four now opens with a "Use when ..." sentence. The routing hook and the contract check that read the new frontmatter landed in v1.94.6.0 (lane A, off by default; see docs/skill-routing-and-contracts.md), so the "Done means" list now also names `gstack-skill-contract check` and `waive`. + +### Changed +- **Descriptions**: the first sentence of each is now a when-clause, for example /investigate's "Use when a test fails, code errors or output is wrong: find the root cause, then fix it." Hosts that show the full description also get the longer list of situations. The catalog stays within its existing budget. +- **Triggers**: /investigate, /review and /verify list the prompts people actually type, such as "test fails", "look over my diff" and "make it green". /qa keeps its three triggers because its size cap had 9 bytes of room. + +### Added +- **`contract:` frontmatter** (frozen v1 schema) on /investigate (the DEBUG REPORT labels, and a test or repro run after the last edit) and /review (one Agent dispatch per selected specialist, the adversarial pass, the Pre-Landing Review report). `{{CONTRACT_DONE_MEANS}}` (`scripts/resolvers/contract-done-means.ts`) renders each block as a short "Done means" list at the top of the skill, so the list and the block can't drift. + +### Tests +- `test/skill-descriptions.test.ts`: each of the four opens with a when-clause within the catalog, lead and host caps; triggers are short and don't overlap; ten sample prompts match exactly the intended skill; every template's contract block passes the v1 schema and names markers the skill text actually prints; the renderer covers each item kind; and malformed blocks fail with the file and the fix. + ## [1.94.21.0] - 2026-11-03 - **A stacked PR's /ship receipt now carries even when the child merged main before its receipt was written (GST-1214).** After the parent squash-landed and main was merged in again, `gstack-ship-receipt carry` compared the range from the parent's head, which still held every file the child's first main merge brought in, and refused with `RECEIPT_CARRY_PRODUCT_CHANGED` naming those files. The carry now leaves out paths whose content at the reviewed head is main's, unless HEAD changed them. A real edit to one of those files still refuses. diff --git a/VERSION b/VERSION index 098e7bdd3..23b0a03fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.21.0 +1.94.22.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 96ec3a476..d442974f0 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.21.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.22.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/gstack/llms.txt b/gstack/llms.txt index 3dbe42205..ec8a7721d 100644 --- a/gstack/llms.txt +++ b/gstack/llms.txt @@ -37,7 +37,7 @@ Conventions: - [/gstack-upgrade](gstack-upgrade/SKILL.md): Upgrade gstack to the latest version. - [/guard](guard/SKILL.md): Full safety mode: destructive command warnings + directory-scoped edits. - [/health](health/SKILL.md): Code quality dashboard. -- [/investigate](investigate/SKILL.md): Systematic debugging with root cause investigation. +- [/investigate](investigate/SKILL.md): Use when a test fails, code errors or output is wrong: find the root cause, then fix it. - [/ios-clean](ios-clean/SKILL.md): Remove the DebugBridge SPM package and all #if DEBUG wiring from an iOS app. - [/ios-design-review](ios-design-review/SKILL.md): Visual design audit for iOS apps on real hardware. - [/ios-fix](ios-fix/SKILL.md): Autonomous iOS bug fixer. @@ -56,10 +56,10 @@ Conventions: - [/plan-devex-review](plan-devex-review/SKILL.md): Interactive developer experience plan review. - [/plan-eng-review](plan-eng-review/SKILL.md): Eng manager-mode plan review. - [/plan-tune](plan-tune/SKILL.md): Self-tuning question sensitivity + developer psychographic for gstack (v1: observational). -- [/qa](qa/SKILL.md): Test the site, API or CLI and fix the bugs it finds. +- [/qa](qa/SKILL.md): Use when you test the site, API or CLI and fix bugs. - [/qa-only](qa-only/SKILL.md): Report browser/API/CLI/job/worker/webhook bugs. - [/retro](retro/SKILL.md): Weekly engineering retrospective. -- [/review](review/SKILL.md): Review this PR's diff. +- [/review](review/SKILL.md): Use when you review this PR's diff. - [/scrape](scrape/SKILL.md): Pull data from a web page through the Aside browser — your real, already signed-in sessions. - [/setup-browser-cookies](setup-browser-cookies/SKILL.md): Import cookies from your real Chromium browser into the headless browse session. - [/setup-deploy](setup-deploy/SKILL.md): Configure deployment settings for /land-and-deploy. @@ -71,7 +71,7 @@ Conventions: - [/test-audit](test-audit/SKILL.md): Find low-value or duplicate tests and the test-only code they keep alive. - [/ubicloud](ubicloud/SKILL.md): Run a heavy suite, build or eval on an ephemeral 16-vCPU Ubicloud VM: TTL-named VMs, a reaper, quota preflight and a hang watchdog. - [/unfreeze](unfreeze/SKILL.md): Clear the freeze boundary set by /freeze, allowing edits to all directories again. -- [/verify](verify/SKILL.md): Build loop while coding: run the repo's checks and the tests /ship would pick, fix, repeat until green. +- [/verify](verify/SKILL.md): Use when coding: run the checks and tests /ship picks until green. - [/write-skill](write-skill/SKILL.md): Write a new agent skill or improve one without bloat: the edit ladder, levels, drift and size checks, full diets. ## Browse Commands diff --git a/investigate/SKILL.md b/investigate/SKILL.md index cb6a50df2..191fe7e05 100644 --- a/investigate/SKILL.md +++ b/investigate/SKILL.md @@ -2,7 +2,7 @@ name: investigate preamble-tier: 2 version: 1.0.0 -description: Systematic debugging with root cause investigation. (gstack) +description: "Use when a test fails, code errors or output is wrong: find the root cause, then fix it. (gstack)" allowed-tools: - Bash - Read @@ -15,9 +15,29 @@ allowed-tools: triggers: - debug this - fix this bug + - test fails + - failing test - why is this broken - - root cause analysis + - this doesn't work + - build is broken + - stack trace - investigate this error + - root cause analysis +contract: + version: 1 + items: + - id: debug-report + kind: report_labels + heading: DEBUG REPORT + labels: + - Symptom + - Root cause + - Fix + - Evidence + - Regression test + - Status + - id: check-after-last-edit + kind: check_after_last_edit hooks: PreToolUse: - matcher: "Edit" @@ -61,6 +81,15 @@ gbrain: +## Done means + +Finish only when each of these exists (contract v1): + +- `debug-report`: a `DEBUG REPORT` with Symptom, Root cause, Fix, Evidence, Regression test and Status filled in (`none: ` where the skill's own stop rule leaves one empty). +- `check-after-last-edit`: a test or repro command run after your last edit. + +`gstack-skill-contract check` reads them from the transcript (docs/skill-routing-and-contracts.md). For an item that truly does not apply, run `gstack-skill-contract waive --reason ""` (this session only; it counts as a miss). + ## Shared preamble (load first) This skill follows gstack shared preamble `4e82fa4c839f` (AskUserQuestion format, voice, writing, completeness and completion rules). If your Read of that file, headed by this id, is still in context, do not read it again; otherwise Read `~/.claude/skills/gstack/investigate/sections/preamble.md` in full before the command below. A section or subsection here with a shared heading replaces the shared one. diff --git a/investigate/SKILL.md.tmpl b/investigate/SKILL.md.tmpl index c7469537d..a9f7b2ecb 100644 --- a/investigate/SKILL.md.tmpl +++ b/investigate/SKILL.md.tmpl @@ -3,10 +3,10 @@ name: investigate preamble-tier: 2 version: 1.0.0 description: | - Systematic debugging with root cause investigation. Iron Law: no fixes - without root cause. - Use when asked to "debug this", "fix this bug", "why is this broken", - "investigate this error", or "root cause analysis". (gstack) + Use when a test fails, code errors or output is wrong: find the root cause, then fix it. + Systematic debugging; Iron Law: no fixes without root cause. Fits a failing + build, test, script, service or config, an error message or stack trace, + wrong or missing output, or broken data, before any edit that fixes it. (gstack) allowed-tools: - Bash - Read @@ -19,9 +19,29 @@ allowed-tools: triggers: - debug this - fix this bug + - test fails + - failing test - why is this broken - - root cause analysis + - this doesn't work + - build is broken + - stack trace - investigate this error + - root cause analysis +contract: + version: 1 + items: + - id: debug-report + kind: report_labels + heading: DEBUG REPORT + labels: + - Symptom + - Root cause + - Fix + - Evidence + - Regression test + - Status + - id: check-after-last-edit + kind: check_after_last_edit hooks: PreToolUse: - matcher: "Edit" @@ -63,6 +83,8 @@ gbrain: render_as: "## Recent eureka moments (cross-project)" --- +{{CONTRACT_DONE_MEANS}} + {{PREAMBLE}} # Systematic Debugging diff --git a/package.json b/package.json index eb743f2a0..672f1eebb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.21", + "version": "1.94.22", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/qa/SKILL.md b/qa/SKILL.md index 5a348f9c7..5c8b6ab91 100644 --- a/qa/SKILL.md +++ b/qa/SKILL.md @@ -2,7 +2,7 @@ name: qa preamble-tier: 4 version: 2.0.0 -description: Test the site, API or CLI and fix the bugs it finds. (gstack) +description: Use when you test the site, API or CLI and fix bugs. (gstack) allowed-tools: - Bash - Read diff --git a/qa/SKILL.md.tmpl b/qa/SKILL.md.tmpl index 7c411d8c0..7bc6706b1 100644 --- a/qa/SKILL.md.tmpl +++ b/qa/SKILL.md.tmpl @@ -3,9 +3,9 @@ name: qa preamble-tier: 4 version: 2.0.0 description: | - Test the site, API or CLI and fix the bugs it finds. + Use when you test the site, API or CLI and fix bugs. Covers browser/API/CLI/job/worker/webhook bugs and commits verified fixes atomically. - Use when asked to "qa", "QA", "qa test this", "test this site", "find bugs", "find bugs on site", + Fits "qa", "QA", "qa test this", "test this site", "find bugs", "find bugs on site", "test and fix", or "fix what's broken". Proactively suggest when the user says a feature is ready for testing or asks "does this work?". Three tiers: Quick (critical/high only), diff --git a/review/SKILL.md b/review/SKILL.md index 369f3730a..b01e94d13 100644 --- a/review/SKILL.md +++ b/review/SKILL.md @@ -2,7 +2,7 @@ name: review preamble-tier: 4 version: 1.0.0 -description: Review this PR's diff. (gstack) +description: Use when you review this PR's diff. (gstack) allowed-tools: - Bash - Read @@ -15,13 +15,39 @@ allowed-tools: - WebSearch triggers: - review this pr + - look over my diff - code review - check my diff + - review my changes - pre-landing review +contract: + version: 1 + items: + - id: specialists-dispatched + kind: selected_specialists_dispatched + marker: "Dispatching N specialists:" + - id: adversarial-pass + kind: agent_dispatch + match: Think like an attacker and a chaos engineer + - id: review-report + kind: report_labels + heading: Pre-Landing Review + labels: + - Gate edits --- +## Done means + +Finish only when each of these exists (contract v1): + +- `specialists-dispatched`: the specialist selection printed (`Dispatching N specialists:` ...), then one Agent dispatch per specialist it names (none when a small diff skips them). +- `adversarial-pass`: an Agent dispatch for the adversarial pass, its prompt including "Think like an attacker and a chaos engineer". +- `review-report`: a `Pre-Landing Review` with Gate edits filled in (`none: ` where the skill's own stop rule leaves one empty). + +`gstack-skill-contract check` reads them from the transcript (docs/skill-routing-and-contracts.md). For an item that truly does not apply, run `gstack-skill-contract waive --reason ""` (this session only; it counts as a miss). + ## Shared preamble (load first) This skill follows gstack shared preamble `4e82fa4c839f` (AskUserQuestion format, voice, writing, completeness and completion rules). If your Read of that file, headed by this id, is still in context, do not read it again; otherwise Read `~/.claude/skills/gstack/review/sections/preamble.md` in full before the command below. A section or subsection here with a shared heading replaces the shared one. diff --git a/review/SKILL.md.tmpl b/review/SKILL.md.tmpl index 60ca668bd..7ae3e4eef 100644 --- a/review/SKILL.md.tmpl +++ b/review/SKILL.md.tmpl @@ -3,10 +3,11 @@ name: review preamble-tier: 4 version: 1.0.0 description: | - Review this PR's diff. Checks it against the base branch for SQL safety, LLM trust - boundary violations, conditional side effects, and other structural issues. Use when - asked to "review this PR", "code review", "pre-landing review", or "check my diff". - Proactively suggest when the user is about to merge or land code changes. (gstack) + Use when you review this PR's diff. + Checks it for SQL safety, LLM trust boundaries, conditional side effects and + bugs that pass CI and break production. Fits "look over my diff", "review my + changes", "code review", "check my diff" or "pre-landing review". + Proactively suggest before a merge. (gstack) codex_default_prompt: "Use $gstack-review to review this diff against its base before it lands: find the bugs that pass CI and break in production, fix the clear ones, and list what remains." allowed-tools: - Bash @@ -20,11 +21,29 @@ allowed-tools: - WebSearch triggers: - review this pr + - look over my diff - code review - check my diff + - review my changes - pre-landing review +contract: + version: 1 + items: + - id: specialists-dispatched + kind: selected_specialists_dispatched + marker: "Dispatching N specialists:" + - id: adversarial-pass + kind: agent_dispatch + match: Think like an attacker and a chaos engineer + - id: review-report + kind: report_labels + heading: Pre-Landing Review + labels: + - Gate edits --- +{{CONTRACT_DONE_MEANS}} + {{PREAMBLE}} {{BASE_BRANCH_DETECT}} diff --git a/scripts/resolvers/contract-done-means.ts b/scripts/resolvers/contract-done-means.ts new file mode 100644 index 000000000..ef98c9b0d --- /dev/null +++ b/scripts/resolvers/contract-done-means.ts @@ -0,0 +1,142 @@ +/** + * {{CONTRACT_DONE_MEANS}}: a skill's `contract:` frontmatter block (GST-1088, frozen v1 + * schema) rendered as a short "Done means" list at the top of the body, so the + * outputs the model reads as required and the block the Stop check reads cannot + * drift. parseSkillContract is the schema; test/skill-descriptions.test.ts runs it + * over every template, so a malformed block fails CI instead of failing open. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { TemplateContext } from './types'; +import { getHostConfig } from '../../hosts/index'; + +const ROOT = path.resolve(import.meta.dir, '..', '..'); +const code = (s: string) => '\u0060' + s + '\u0060'; + +export type ContractItem = + | { id: string; kind: 'report_labels'; heading: string; labels: string[] } + | { id: string; kind: 'check_after_last_edit' } + | { id: string; kind: 'selected_specialists_dispatched'; marker: string } + | { id: string; kind: 'agent_dispatch'; match: string }; + +export interface SkillContract { + version: 1; + items: ContractItem[]; +} + +const KIND_FIELDS: Record = { + report_labels: ['heading', 'labels'], + check_after_last_edit: [], + selected_specialists_dispatched: ['marker'], + agent_dispatch: ['match'], +}; + +// The resolver and section that render the step each kind checks today. A host that suppresses the resolver, or a loadout that excludes the section, never shows that step, so its items are left out of the list there. +const BACKED_BY: Partial> = { + selected_specialists_dispatched: { resolver: 'REVIEW_ARMY', section: 'review-army' }, + agent_dispatch: { resolver: 'ADVERSARIAL_STEP', section: 'adversarial' }, +}; + +const PLACEHOLDER = /^\[.*\]$|^<.*>$|^\.\.\.$|^(todo|tbd|n\/a)$/i; + +function fail(where: string, problem: string): never { + throw new Error('CONTRACT_INVALID ' + where + ': ' + problem + '. Fix: edit ' + where + ', then bun run gen:skill-docs'); +} + +const isMap = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const isText = (v: unknown): v is string => typeof v === 'string' && v.trim() !== ''; + +/** Validates a parsed `contract:` value against the frozen v1 schema; where names the template in errors. */ +export function parseSkillContract(value: unknown, where: string): SkillContract { + if (!isMap(value)) fail(where, 'contract must be a mapping { version: 1, items: [...] }'); + for (const key of Object.keys(value)) { + if (key !== 'version' && key !== 'items') fail(where, 'contract has unknown key "' + key + '"'); + } + if (value.version !== 1) fail(where, 'contract version must be 1, got ' + JSON.stringify(value.version)); + if (!Array.isArray(value.items) || value.items.length === 0) fail(where, 'contract items must be a non-empty list'); + const seen = new Set(); + const items = value.items.map((raw: unknown, i: number) => { + const at = 'items[' + i + ']'; + if (!isMap(raw)) fail(where, at + ' must be a mapping'); + const kind = raw.kind as ContractItem['kind']; + if (typeof raw.kind !== 'string' || !Object.hasOwn(KIND_FIELDS, kind)) { + fail(where, at + ' kind ' + JSON.stringify(raw.kind) + ' is not one of ' + Object.keys(KIND_FIELDS).join(', ')); + } + if (typeof raw.id !== 'string' || !/^[a-z][a-z0-9]*(-[a-z0-9]+)*$/.test(raw.id)) { + fail(where, at + ' id must be lowercase kebab-case, got ' + JSON.stringify(raw.id)); + } + if (seen.has(raw.id)) fail(where, at + ' duplicate id "' + raw.id + '"'); + seen.add(raw.id); + for (const key of Object.keys(raw)) { + if (key !== 'id' && key !== 'kind' && !KIND_FIELDS[kind].includes(key)) fail(where, at + ' (' + kind + ') has unknown key "' + key + '"'); + } + for (const field of KIND_FIELDS[kind]) { + if (field !== 'labels' && !isText(raw[field])) fail(where, at + ' ' + field + ' must be a non-empty string'); + } + if (kind === 'report_labels') { + if (!Array.isArray(raw.labels) || raw.labels.length === 0) fail(where, at + ' labels must be a non-empty list'); + for (const label of raw.labels as unknown[]) { + if (!isText(label)) fail(where, at + ' labels must be non-empty strings'); + if (PLACEHOLDER.test(label.trim())) fail(where, at + ' label ' + JSON.stringify(label) + ' is a placeholder'); + } + } + return raw as ContractItem; + }); + return { version: 1, items }; +} + +/** The template's contract, or null without one; required makes a missing block an error. */ +export function contractFromTemplate(tmplContent: string, where: string, opts: { required?: boolean } = {}): SkillContract | null { + const block = tmplContent.replace(/\r\n/g, '\n').match(/^---\n([\s\S]*?)\n---(?:\n|$)/); + if (!block || !/^contract:/m.test(block[1])) { + if (opts.required) fail(where, '{{CONTRACT_DONE_MEANS}} needs a contract block in the frontmatter'); + return null; + } + let frontmatter: unknown; + try { + frontmatter = Bun.YAML.parse(block[1]); + } catch (error) { + fail(where, 'frontmatter is not valid YAML (' + (error instanceof Error ? error.message : String(error)) + ')'); + } + return parseSkillContract(isMap(frontmatter) ? frontmatter.contract : undefined, where); +} + +const list = (xs: string[]) => xs.length < 2 ? xs.join('') : xs.slice(0, -1).join(', ') + ' and ' + xs[xs.length - 1]; + +function plainWords(item: ContractItem): string { + switch (item.kind) { + case 'report_labels': return 'a ' + code(item.heading) + ' with ' + list(item.labels) + ' filled in (' + code('none: ') + ' where the skill\'s own stop rule leaves one empty)'; + case 'check_after_last_edit': return 'a test or repro command run after your last edit'; + case 'selected_specialists_dispatched': return 'the specialist selection printed (' + code(item.marker) + ' ...), then one Agent dispatch per specialist it names (none when a small diff skips them)'; + case 'agent_dispatch': return 'an Agent dispatch for the ' + item.id.replace(/-/g, ' ') + ', its prompt including "' + item.match + '"'; + } +} + +/** Whether bin/gstack-skill-contract ships. The PR that adds it flips this, regenerates and re-pins; test/skill-descriptions.test.ts fails while the two disagree. */ +export const CONTRACT_CHECKER_SHIPPED = true; + +/** The list; checker adds the check and waive commands, which only a tree that ships the checker may mention. */ +export function renderDoneMeans(contract: SkillContract, checker = CONTRACT_CHECKER_SHIPPED): string { + if (contract.items.length === 0) return ''; + return [ + '## Done means', + '', + 'Finish only when each of these exists (contract v1):', + '', + ...contract.items.map(item => '- ' + code(item.id) + ': ' + plainWords(item) + '.'), + ...(checker ? ['', code('gstack-skill-contract check') + ' reads them from the transcript (docs/skill-routing-and-contracts.md). ' + + 'For an item that truly does not apply, run ' + code('gstack-skill-contract waive --reason ""') + ' (this session only; it counts as a miss).'] : []), + ].join('\n'); +} + +export function generateContractDoneMeans(ctx: TemplateContext): string { + const where = path.relative(ROOT, ctx.tmplPath).split(path.sep).join('/'); + const contract = contractFromTemplate(fs.readFileSync(ctx.tmplPath, 'utf-8'), where, { required: true })!; + const suppressed = new Set(getHostConfig(ctx.host).suppressedResolvers ?? []); + const shown = (item: ContractItem) => { + const backing = BACKED_BY[item.kind]; + return !backing || (!suppressed.has(backing.resolver) && !ctx.modules?.excluded.has('section:' + ctx.skillName + '/' + backing.section)); + }; + return renderDoneMeans({ ...contract, items: contract.items.filter(shown) }); +} diff --git a/scripts/resolvers/index.ts b/scripts/resolvers/index.ts index e205fcae9..2fd070d74 100644 --- a/scripts/resolvers/index.ts +++ b/scripts/resolvers/index.ts @@ -58,6 +58,7 @@ import { generateShipDriverStops } from './ship-driver'; import { generateQAScope, generateQAExploratory, generateQAFunctional, generateQAResource, generateQAReview, generateQAReviewPreflight, generateQAMethodReads } from './qa'; import { generateVoiceTriggerRouting } from './voice-triggers'; import { generateTranscriptDiscovery, generateTranscriptSources } from './transcripts'; +import { generateContractDoneMeans } from './contract-done-means'; export const RESOLVERS: Record = { AUTOPLAN_PUBLICATION_HOOK: generateAutoplanPublicationHook, @@ -197,4 +198,5 @@ export const RESOLVERS: Record = { SECTION_INDEX, TRANSCRIPT_SOURCES: generateTranscriptSources, TRANSCRIPT_DISCOVERY: generateTranscriptDiscovery, + CONTRACT_DONE_MEANS: generateContractDoneMeans, }; diff --git a/test/fixtures/skill-md-size-allowlist.json b/test/fixtures/skill-md-size-allowlist.json index a0627d4b9..fe34e981a 100644 --- a/test/fixtures/skill-md-size-allowlist.json +++ b/test/fixtures/skill-md-size-allowlist.json @@ -6,52 +6,52 @@ "agy/gstack-plan-design-review": 109107, "agy/gstack-plan-devex-review": 114948, "agy/gstack-plan-eng-review": 143581, - "agy/gstack-review": 104182, + "agy/gstack-review": 104728, "codex/gstack-autoplan": 108400, "codex/gstack-plan-design-review": 110464, "codex/gstack-plan-devex-review": 116456, "codex/gstack-plan-eng-review": 144746, - "codex/gstack-review": 103705, + "codex/gstack-review": 104251, "copilot/gstack-autoplan": 109100, "copilot/gstack-plan-design-review": 109373, "copilot/gstack-plan-devex-review": 114410, "copilot/gstack-plan-eng-review": 143746, - "copilot/gstack-review": 103763, + "copilot/gstack-review": 104309, "cursor/gstack-autoplan": 108332, "cursor/gstack-plan-design-review": 107915, "cursor/gstack-plan-devex-review": 113680, "cursor/gstack-plan-eng-review": 142336, - "cursor/gstack-review": 122053, + "cursor/gstack-review": 122781, "factory/gstack-autoplan": 110610, "factory/gstack-plan-design-review": 111013, "factory/gstack-plan-devex-review": 117043, "factory/gstack-plan-eng-review": 145444, - "factory/gstack-review": 125826, + "factory/gstack-review": 126554, "gbrain/gstack-autoplan": 108739, "gbrain/gstack-plan-eng-review": 124963, "gemini/gstack-autoplan": 108890, "gemini/gstack-plan-design-review": 109201, "gemini/gstack-plan-devex-review": 114238, "gemini/gstack-plan-eng-review": 143579, - "gemini/gstack-review": 103585, + "gemini/gstack-review": 104131, "hermes/gstack-autoplan": 108742, "hermes/gstack-plan-eng-review": 124999, "kiro/gstack-autoplan": 108106, "kiro/gstack-plan-design-review": 107761, "kiro/gstack-plan-devex-review": 113532, "kiro/gstack-plan-eng-review": 142198, - "kiro/gstack-review": 121877, + "kiro/gstack-review": 122605, "openclaw/gstack-autoplan": 108562, "openclaw/gstack-plan-eng-review": 123478, "opencode/gstack-autoplan": 109081, "opencode/gstack-plan-design-review": 109229, "opencode/gstack-plan-devex-review": 114288, "opencode/gstack-plan-eng-review": 143605, - "opencode/gstack-review": 122675, + "opencode/gstack-review": 123403, "slate/gstack-autoplan": 108219, "slate/gstack-plan-design-review": 107838, "slate/gstack-plan-devex-review": 113606, "slate/gstack-plan-eng-review": 142267, - "slate/gstack-review": 121965 + "slate/gstack-review": 122693 } } diff --git a/test/harness-register.test.ts b/test/harness-register.test.ts index 6544b11d1..b483d491c 100644 --- a/test/harness-register.test.ts +++ b/test/harness-register.test.ts @@ -248,7 +248,7 @@ describe('GSTA-135: registered skills get used', () => { } const oc = register(opts(d, 'openclaw', { mode: 'all', skillsDir: path.join(d, 'oc') })); const review = fs.readFileSync(path.join(oc.dest, 'gstack-review', 'SKILL.md'), 'utf8'); - expect(descriptionOf(review)[0]).not.toMatch(/^Use /); + expect(descriptionOf(review)[0]).not.toMatch(/^Use for: /); }, 120_000); test('OpenClaw hybrid is the default: the task skills listed directly, the router for the rest; mode switches refresh and uninstall cleanly', () => { diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index 901100cc1..659255e43 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -490,10 +490,10 @@ do not launch the downstream skill or open a browser.`, gateAfterStop: undefined, // operational multi-STOP skill, like ship }, behavioral: 'plan', - maxSkeletonBytes: 59_073, // GSTA-96 hoist (2026-10-26): lowered from 80_900 by the shared preamble that moved to sections/preamble.md, keeping the pre-hoist headroom of 319 bytes; measured 58,754. Before: Shared-code identity/skip/action rules + critical-severity validation; measured 74,493 (2026-09-17). + v1.91.12.0 merge of #2999 (review clarity repairs: await reads, research alongside dispatch, /review deadline and setup authority, findings sources) with #3002 (guarded state-root lines, plan-check checkpoints); each fit alone; measured 74,881 (2026-10-01). + Oct 8 wave: shared voice directive bans "load-bearing" and adds the reply-language rule, offset by tightening the bounded closer (net +17 bytes per tier-2+ skill); measured 74,884 (2026-10-08). + gate integrity (HONEST_WORK_GATE_INTEGRITY W2): Step 3.5 Diff scans with the shared GATE_SCAN_BLOCK, the Step 4 carry, the four-option `review-gate-disposition` question in 5c, gate record fields in 5.8 and the Gate edits report line; measured 80,054 (2026-10-10). + Oct 10 multi-agent wave PR D2 (sideways review): Step 4's enum-only paragraph becomes the sideways sweep with the forced-CRITICAL list, plus the flake-fix number sentence in the Gate Integrity paragraph; measured skeleton 80,812 (2026-10-10). + maxSkeletonBytes: 60_014, // GST-1088 lane B (#3909, raised from 59,774 to measured, no headroom; owner ruling via GStack Minder 2026-11-03): CONTRACT_CHECKER_SHIPPED=true now that bin/gstack-skill-contract ships (#3912) renders the gstack-skill-contract check/waive sentence under the Done means list (renders/GSTA-256/plan-v3.md line 74, r2-d6 "Done-means list"), +240 bytes; lane C's diet is expected to bring /review back under 59,073; measured 60,014. Before: GST-1088 lane B (raised from 59,073 to measured, no headroom): the contract: frontmatter block and the F1 "Done means" list rendered from it at the top of the body, plus widened triggers; F1 is a gate-rev-2 recommendation on the plan Garry approved with "do it" on gate rev 1 (2026-11-03), contingent on his gate-rev-2 answer, and lane C's diet is expected to bring /review back under 59,073; measured 59,774. Before: GSTA-96 hoist (2026-10-26): lowered from 80_900 by the shared preamble that moved to sections/preamble.md, keeping the pre-hoist headroom of 319 bytes; measured 58,754. Before: Shared-code identity/skip/action rules + critical-severity validation; measured 74,493 (2026-09-17). + v1.91.12.0 merge of #2999 (review clarity repairs: await reads, research alongside dispatch, /review deadline and setup authority, findings sources) with #3002 (guarded state-root lines, plan-check checkpoints); each fit alone; measured 74,881 (2026-10-01). + Oct 8 wave: shared voice directive bans "load-bearing" and adds the reply-language rule, offset by tightening the bounded closer (net +17 bytes per tier-2+ skill); measured 74,884 (2026-10-08). + gate integrity (HONEST_WORK_GATE_INTEGRITY W2): Step 3.5 Diff scans with the shared GATE_SCAN_BLOCK, the Step 4 carry, the four-option `review-gate-disposition` question in 5c, gate record fields in 5.8 and the Gate edits report line; measured 80,054 (2026-10-10). + Oct 10 multi-agent wave PR D2 (sideways review): Step 4's enum-only paragraph becomes the sideways sweep with the forced-CRITICAL list, plus the flake-fix number sentence in the Gate Integrity paragraph; measured skeleton 80,812 (2026-10-10). minUnionBytes: 89_000, // Phase 4 wave 1; measured union 93,357 mustContain: ['confidence', 'P1', 'P2', 'Review Army', 'adversarial'], - maxSizeRatio: 1.3180, // + GSTA-167 variant G part 1 (plan approved by Garry 2026-10-30, autoplan-20261030161754 "Do it", which names about +0.3k prompt tokens per voice): the native adversarial and outside-voice prompts carry the blocking floor and the Severity and Blocks lines, +537 bytes; measured 1.3180 (2026-10-30). Before: 1.3136. GSTA-96 hoist merged with GSTA-92 phase 2 (2026-10-26): both owner-approved raises together, measured 142,550 / 108,523 = 1.3135. Before: Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 132,258 / 108,523 = 1.2187 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 132,233 / 108,523 = 1.2185 (2026-10-07). Merged with the guard area: measured 132,522 / 108,523 = 1.2211 (2026-10-07). + Oct 8 wave: the defensive-security framing and fixture summary mode reach the security specialist and the Red Team (stated once in the dispatch, referenced by the Red Team); measured 133,902 / 108,523 = 1.2339 (2026-10-08). + gate integrity W2 (above); measured 1.282 (2026-10-10). + Oct 10 multi-agent wave PR D2 (sideways review): sideways sweep and flake-fix number in Step 4; `Diff read` and `Undeclared behavior changes` in the review-mode Scope Check (plan-completion section); measured 140,497 / 108,523 = 1.2946 (2026-10-10). + harness lane E (2026-10-26): the lead-judgment stage in the finding merge and the `cause` anchor in the specialist prompts, shared with /ship through scripts/resolvers/review-army.ts; measured 141,949 / 108,523 = 1.3080. + maxSizeRatio: 1.3283, // + GST-1088 lane B (#3909, raised from 1.3261 to measured, no headroom; owner ruling via GStack Minder 2026-11-03): the gstack-skill-contract check/waive sentence that CONTRACT_CHECKER_SHIPPED=true renders (renders/GSTA-256/plan-v3.md line 74, r2-d6 "Done-means list"), +240 bytes; lane C's diet is expected to bring /review back under 1.3180; measured 144,147 / 108,523 = 1.32826, rounded up. Before: + GST-1088 lane B (raised from 1.3180 to measured, no headroom): the contract: frontmatter block and the F1 "Done means" list, contingent on Garry's gate-rev-2 answer; lane C's diet is expected to bring /review back under 1.3180; measured 143,907 / 108,523 = 1.3261. Before: + GSTA-167 variant G part 1 (plan approved by Garry 2026-10-30, autoplan-20261030161754 "Do it", which names about +0.3k prompt tokens per voice): the native adversarial and outside-voice prompts carry the blocking floor and the Severity and Blocks lines, +537 bytes; measured 1.3180 (2026-10-30). Before: 1.3136. GSTA-96 hoist merged with GSTA-92 phase 2 (2026-10-26): both owner-approved raises together, measured 142,550 / 108,523 = 1.3135. Before: Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 132,258 / 108,523 = 1.2187 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 132,233 / 108,523 = 1.2185 (2026-10-07). Merged with the guard area: measured 132,522 / 108,523 = 1.2211 (2026-10-07). + Oct 8 wave: the defensive-security framing and fixture summary mode reach the security specialist and the Red Team (stated once in the dispatch, referenced by the Red Team); measured 133,902 / 108,523 = 1.2339 (2026-10-08). + gate integrity W2 (above); measured 1.282 (2026-10-10). + Oct 10 multi-agent wave PR D2 (sideways review): sideways sweep and flake-fix number in Step 4; `Diff read` and `Undeclared behavior changes` in the review-mode Scope Check (plan-completion section); measured 140,497 / 108,523 = 1.2946 (2026-10-10). + harness lane E (2026-10-26): the lead-judgment stage in the finding merge and the `cause` anchor in the specialist prompts, shared with /ship through scripts/resolvers/review-army.ts; measured 141,949 / 108,523 = 1.3080. }, codex: { skill: 'codex', diff --git a/test/skill-descriptions.test.ts b/test/skill-descriptions.test.ts new file mode 100644 index 000000000..d6eff7b43 --- /dev/null +++ b/test/skill-descriptions.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, test } from 'bun:test'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { sentences } from '../lib/skill-triggers'; +import { CONTRACT_CHECKER_SHIPPED, contractFromTemplate, generateContractDoneMeans, parseSkillContract, renderDoneMeans } from '../scripts/resolvers/contract-done-means'; +import { HOST_PATHS } from '../scripts/resolvers/types'; + +/** + * GST-1088 discovery targets: each says when to use it in the part of its + * description every host shows, and every contract block matches the frozen + * v1 schema, so a malformed block fails here instead of failing open in the + * Stop check. + */ + +const ROOT = join(import.meta.dir, '..'); +const SKILLS = ['investigate', 'review', 'qa', 'verify']; +// splitCatalogDescription cuts a longer Claude lead with "...", which drops the end of the when-clause. +const LEAD_CAP = 200; +// test/catalog-budget.test.ts PER_SKILL_BYTE_CAP: name + Claude description bytes. +const CATALOG_ENTRY_CAP = 260; +// Codex, Gemini, Copilot and agy reject a longer description (hosts/*.ts descriptionLimit). +const DESCRIPTION_CAP = 1024; +const TRIGGER_CAP = 32; +const BT = '\u0060'; + +const fix = (skill: string) => 'Fix: edit ' + skill + '/SKILL.md.tmpl, then bun run gen:skill-docs'; +const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8').replace(/\r\n/g, '\n'); + +function frontmatter(rel: string): Record { + const m = read(rel).match(/^---\n([\s\S]*?)\n---\n/); + if (!m) throw new Error(rel + ': no frontmatter'); + return Bun.YAML.parse(m[1]) as Record; +} + +const body = (rel: string) => read(rel).replace(/^---\n[\s\S]*?\n---\n/, ''); +const triggersOf = (skill: string) => (frontmatter(skill + '/SKILL.md.tmpl').triggers ?? []) as string[]; + +describe('discovery-target descriptions (GST-1088)', () => { + for (const skill of SKILLS) { + const tmpl = skill + '/SKILL.md.tmpl'; + + test(skill + ': the template description opens with a "Use when ..." sentence', () => { + const first = sentences(String(frontmatter(tmpl).description ?? ''))[0] ?? ''; + expect(/^Use when\b/.test(first), tmpl + ': the first sentence is "' + first.slice(0, 80) + '", not a "Use when ..." clause. ' + fix(skill)).toBe(true); + }); + + test(skill + ': the Claude catalog entry keeps the when-clause within its caps', () => { + const rel = skill + '/SKILL.md'; + const fm = frontmatter(rel); + const description = String(fm.description ?? ''); + const lead = description.replace(/\s*\(gstack\)\s*$/, ''); + expect(/^Use when\b/.test(lead), rel + ': the catalog description "' + lead.slice(0, 80) + '" does not start with "Use when". ' + fix(skill)).toBe(true); + expect(lead.length, rel + ': the catalog lead is ' + lead.length + ' chars against the ' + LEAD_CAP + '-char cap. ' + fix(skill)).toBeLessThanOrEqual(LEAD_CAP); + const entry = Buffer.byteLength(String(fm.name)) + Buffer.byteLength(description); + expect(entry, rel + ': name + description is ' + entry + ' bytes against the ' + CATALOG_ENTRY_CAP + '-byte cap. ' + fix(skill)).toBeLessThanOrEqual(CATALOG_ENTRY_CAP); + }); + + test(skill + ': the full description fits every host limit', () => { + const full = String(frontmatter(tmpl).description ?? '').trim(); + expect(full.length, tmpl + ': the description is ' + full.length + ' chars against the ' + DESCRIPTION_CAP + '-char cap. ' + fix(skill)).toBeLessThanOrEqual(DESCRIPTION_CAP); + }); + + test(skill + ': triggers are short and distinct', () => { + const own = triggersOf(skill); + expect(own.length, tmpl + ': no triggers. ' + fix(skill)).toBeGreaterThan(0); + expect(new Set(own).size, tmpl + ': duplicate triggers. ' + fix(skill)).toBe(own.length); + for (const t of own) { + expect(t.length, tmpl + ': trigger "' + t + '" is ' + t.length + ' chars against the ' + TRIGGER_CAP + '-char cap. ' + fix(skill)).toBeLessThanOrEqual(TRIGGER_CAP); + expect(t, tmpl + ': trigger "' + t + '" must be lowercase. ' + fix(skill)).toBe(t.toLowerCase()); + for (const other of SKILLS.filter(s => s !== skill)) { + const clash = triggersOf(other).find(o => o.includes(t) || t.includes(o)); + expect(clash, tmpl + ': trigger "' + t + '" overlaps ' + other + "'s \"" + clash + '". ' + fix(skill)).toBeUndefined(); + } + } + }); + } + + const PROMPTS: Array<[string, string]> = [ + ['this test fails, fix it', 'investigate'], + ['why is this broken?', 'investigate'], + ['the build is broken, here is the stack trace', 'investigate'], + ['review this PR', 'review'], + ['can you look over my diff', 'review'], + ['review my changes before I merge', 'review'], + ['test the site and fix what you find', 'qa'], + ['can you qa test this page', 'qa'], + ['run the checks and make it green', 'verify'], + ['verify my change', 'verify'], + ]; + for (const [prompt, want] of PROMPTS) { + test('untold prompt "' + prompt + '" carries a ' + want + ' trigger and no other target trigger', () => { + const p = prompt.toLowerCase(); + const hits = SKILLS.filter(s => triggersOf(s).some(t => p.includes(t))); + expect(hits, '"' + prompt + '" matches [' + hits.join(', ') + '], want [' + want + ']. ' + fix(want)).toEqual([want]); + }); + } +}); + +describe('contract blocks (frozen v1 schema)', () => { + const templates = readdirSync(ROOT, { withFileTypes: true }) + .filter(d => d.isDirectory() && existsSync(join(ROOT, d.name, 'SKILL.md.tmpl'))) + .map(d => d.name) + .sort(); + + test('investigate and review declare contracts', () => { + for (const skill of ['investigate', 'review']) { + const tmpl = skill + '/SKILL.md.tmpl'; + expect(contractFromTemplate(read(tmpl), tmpl), tmpl + ': no contract block. ' + fix(skill)).not.toBeNull(); + } + }); + + for (const skill of templates) { + const tmpl = skill + '/SKILL.md.tmpl'; + const contract = contractFromTemplate(read(tmpl), tmpl); + if (!contract) { + test(skill + ': no contract, no Done-means placeholder', () => { + expect(read(tmpl).includes('{{CONTRACT_DONE_MEANS}}'), tmpl + ': {{CONTRACT_DONE_MEANS}} without a contract block. ' + fix(skill)).toBe(false); + }); + continue; + } + + test(skill + ': the body carries the Done-means placeholder and the render carries the list', () => { + expect(body(tmpl).includes('{{CONTRACT_DONE_MEANS}}'), tmpl + ': a contract block needs {{CONTRACT_DONE_MEANS}} at the top of the body. ' + fix(skill)).toBe(true); + expect(read(skill + '/SKILL.md').includes(renderDoneMeans(contract)), skill + '/SKILL.md: the Done-means list is stale. ' + fix(skill)).toBe(true); + }); + + test(skill + ': every contract marker appears in the skill text', () => { + const sectionDir = join(ROOT, skill, 'sections'); + const sections = existsSync(sectionDir) + ? readdirSync(sectionDir).filter(f => f.endsWith('.md')).map(f => read(skill + '/sections/' + f)) + : []; + const text = [body(tmpl), ...sections].join('\n'); + for (const item of contract.items) { + const markers = item.kind === 'report_labels' ? [item.heading, ...item.labels.map(l => l + ':')] + : item.kind === 'selected_specialists_dispatched' ? [item.marker] + : item.kind === 'agent_dispatch' ? [item.match] + : []; + for (const marker of markers) { + expect(text.includes(marker), tmpl + ': contract item ' + item.id + ' needs "' + marker + '", which the skill text never prints. ' + fix(skill)).toBe(true); + } + } + }); + } +}); + +describe('contract-done-means renderer', () => { + const all = parseSkillContract({ + version: 1, + items: [ + { id: 'debug-report', kind: 'report_labels', heading: 'DEBUG REPORT', labels: ['Symptom', 'Root cause', 'Status'] }, + { id: 'check-after-last-edit', kind: 'check_after_last_edit' }, + { id: 'specialists-dispatched', kind: 'selected_specialists_dispatched', marker: 'Dispatching N specialists:' }, + { id: 'adversarial-pass', kind: 'agent_dispatch', match: 'chaos engineer' }, + ], + }, 'fixture/SKILL.md.tmpl'); + const out = renderDoneMeans(all, true); + + test('names each item kind in plain words', () => { + expect(out).toContain('## Done means'); + expect(out).toContain('- ' + BT + 'debug-report' + BT + ': a ' + BT + 'DEBUG REPORT' + BT + ' with Symptom, Root cause and Status filled in (' + BT + 'none: ' + BT + " where the skill's own stop rule leaves one empty)."); + expect(out).toContain('- ' + BT + 'check-after-last-edit' + BT + ': a test or repro command run after your last edit.'); + expect(out).toContain('- ' + BT + 'specialists-dispatched' + BT + ': the specialist selection printed (' + BT + 'Dispatching N specialists:' + BT + ' ...), then one Agent dispatch per specialist it names (none when a small diff skips them).'); + expect(out).toContain('- ' + BT + 'adversarial-pass' + BT + ': an Agent dispatch for the adversarial pass, its prompt including "chaos engineer".'); + }); + + test('points to check and waive in the routing-and-contracts doc, never the off switch', () => { + expect(out).toContain(BT + 'gstack-skill-contract check' + BT); + expect(out).toContain(BT + 'gstack-skill-contract waive --reason ""' + BT); + expect(out).toContain('docs/skill-routing-and-contracts.md'); + expect(out).not.toContain('skill_contracts'); + }); + + const one = (item: Record) => ({ version: 1, items: [item] }); + const MALFORMED: Array<[string, unknown, RegExp]> = [ + ['not a mapping', 'yes', /must be a mapping/], + ['wrong version', { version: 2, items: [{ id: 'a', kind: 'check_after_last_edit' }] }, /version must be 1/], + ['no items', { version: 1, items: [] }, /items must be a non-empty list/], + ['unknown top-level key', { version: 1, items: [{ id: 'a', kind: 'check_after_last_edit' }], extra: 1 }, /unknown key "extra"/], + ['unknown kind', one({ id: 'a', kind: 'vibes' }), /kind "vibes" is not one of/], + ['bad id', one({ id: 'Debug Report', kind: 'check_after_last_edit' }), /id must be lowercase kebab-case/], + ['duplicate id', { version: 1, items: [{ id: 'a', kind: 'check_after_last_edit' }, { id: 'a', kind: 'agent_dispatch', match: 'x' }] }, /duplicate id "a"/], + ['labels missing', one({ id: 'a', kind: 'report_labels', heading: 'R' }), /labels must be a non-empty list/], + ['placeholder label', one({ id: 'a', kind: 'report_labels', heading: 'R', labels: ['[what the user observed]'] }), /placeholder/], + ['heading missing', one({ id: 'a', kind: 'report_labels', labels: ['Symptom'] }), /heading must be a non-empty string/], + ['marker missing', one({ id: 'a', kind: 'selected_specialists_dispatched' }), /marker must be a non-empty string/], + ['match missing', one({ id: 'a', kind: 'agent_dispatch' }), /match must be a non-empty string/], + ['field from another kind', one({ id: 'a', kind: 'check_after_last_edit', match: 'x' }), /unknown key "match"/], + ]; + for (const [name, value, want] of MALFORMED) { + test('a malformed block fails loudly: ' + name, () => { + let message = ''; + try { parseSkillContract(value, 'demo/SKILL.md.tmpl'); } catch (e) { message = (e as Error).message; } + expect(message).toMatch(want); + expect(message).toContain('CONTRACT_INVALID demo/SKILL.md.tmpl'); + expect(message).toContain('Fix: edit demo/SKILL.md.tmpl, then bun run gen:skill-docs'); + }); + } + + test('check and waive appear only once bin/gstack-skill-contract ships', () => { + const shipped = existsSync(join(ROOT, 'bin', 'gstack-skill-contract')); + expect(CONTRACT_CHECKER_SHIPPED, 'CONTRACT_CHECKER_SHIPPED in scripts/resolvers/contract-done-means.ts must be ' + shipped + ' while bin/gstack-skill-contract ' + (shipped ? 'exists' : 'is missing') + '. Fix: set it, then bun run gen:skill-docs').toBe(shipped); + for (const skill of ['investigate', 'review']) { + expect(read(skill + '/SKILL.md').includes('gstack-skill-contract'), skill + '/SKILL.md names gstack-skill-contract while bin/gstack-skill-contract ' + (shipped ? 'exists' : 'is missing') + '. Fix: bun run gen:skill-docs').toBe(shipped); + } + if (shipped) expect(existsSync(join(ROOT, 'docs', 'skill-routing-and-contracts.md')), 'docs/skill-routing-and-contracts.md is missing').toBe(true); + expect(renderDoneMeans(all, false)).not.toContain('gstack-skill-contract'); + }); + + test('a host that suppresses, or a loadout that excludes, the step an item checks leaves that item out', () => { + const render = (host: string, excluded: string[] = []) => generateContractDoneMeans({ skillName: 'review', tmplPath: join(ROOT, 'review', 'SKILL.md.tmpl'), host, paths: HOST_PATHS[host], modules: excluded.length ? { name: 'test', excluded: new Set(excluded) } : undefined } as unknown as Parameters[0]); + const ids = (host: string, excluded: string[] = []) => [...render(host, excluded).matchAll(/^- \u0060([a-z-]+)\u0060:/gm)].map(m => m[1]); + expect(ids('claude', ['section:review/review-army', 'section:review/adversarial'])).toEqual(['review-report']); + expect(ids('claude')).toEqual(['specialists-dispatched', 'adversarial-pass', 'review-report']); + expect(ids('codex')).toEqual(['adversarial-pass', 'review-report']); + expect(ids('hermes')).toEqual(['review-report']); + expect(renderDoneMeans({ version: 1, items: [] })).toBe(''); + }); + + test('a placeholder with no contract block fails the render', () => { + expect(() => contractFromTemplate('---\nname: demo\n---\n{{CONTRACT_DONE_MEANS}}\n', 'demo/SKILL.md.tmpl', { required: true })) + .toThrow(/CONTRACT_INVALID demo\/SKILL.md.tmpl: \{\{CONTRACT_DONE_MEANS\}\} needs a contract block/); + }); + + test('invalid frontmatter YAML fails the render', () => { + expect(() => contractFromTemplate('---\nname: demo\ncontract: [unclosed\n---\n', 'demo/SKILL.md.tmpl')) + .toThrow(/CONTRACT_INVALID demo\/SKILL.md.tmpl: frontmatter is not valid YAML/); + }); +}); diff --git a/verify/SKILL.md b/verify/SKILL.md index ea9e4c937..b321341dc 100644 --- a/verify/SKILL.md +++ b/verify/SKILL.md @@ -2,11 +2,13 @@ name: verify preamble-tier: 1 version: 0.1.0 -description: "Build loop while coding: run the repo's checks and the tests /ship would pick, fix, repeat until green. (gstack)" +description: "Use when coding: run the checks and tests /ship picks until green. (gstack)" triggers: - verify my change - run the checks - make the tests pass + - make it green + - is it green - check my work before ship allowed-tools: - Bash diff --git a/verify/SKILL.md.tmpl b/verify/SKILL.md.tmpl index 8f21e5625..78bc39853 100644 --- a/verify/SKILL.md.tmpl +++ b/verify/SKILL.md.tmpl @@ -3,14 +3,16 @@ name: verify preamble-tier: 1 version: 0.1.0 description: | - Build loop while coding: run the repo's checks and the tests /ship would - pick, fix, repeat until green. Never green when no test covers a change. - Use when asked to "verify", "check my work", "run the checks", "make it - green", or after each change before /ship. (gstack) + Use when coding: run the checks and tests /ship picks until green. + Build loop: run the repo's checks and the tests /ship would pick, fix, repeat. + Never green when no test covers a change. Fits "verify", "check my work", "run + the checks", "make it green", or after each change before /ship. (gstack) triggers: - verify my change - run the checks - make the tests pass + - make it green + - is it green - check my work before ship allowed-tools: - Bash From b3e9e5d32c2e566a93e839cdb83e42e796aa433b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 07:32:05 -0800 Subject: [PATCH 4/5] v1.94.23.0 fix(pregate): strays matches only documented scratch names; literals ignores a named literal the branch only adds (GST-1211) (#3944) * fix(pregate): strays matches only the documented scratch names; literals ignores a named literal the branch only adds (GST-1211) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): literals reports every removed value when the name still has one, so a delete-plus-edit keeps the edited site's old value (GST-1211 review) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 11 ++++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/pregate.md | 7 +++++-- docs/troubleshooting.md | 4 +++- lib/pregate/checks/literals.ts | 17 +++++++++++----- lib/pregate/checks/strays.ts | 2 +- package.json | 2 +- test/pregate-selection.test.ts | 37 ++++++++++++++++++++++++++++++++++ 9 files changed, 72 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e23f5c500..b7b148dc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.94.23.0] - 2026-11-03 + +**Two `gstack-pregate` preflight false positives fixed (GST-1211).** Both stopped /ship on real work in gbrain-evals. + +### Fixed +- **`strays`:** the scratch-name check matched three prefixes the docs never listed (`temp`, `debug-` and `probe`), so real files like `probe.py` and `temporal-graph-shipped.test.ts` failed preflight. It now matches only the documented names: `zz-*`, `tmp*`, `scratch*`, plus editor leftovers (`*.orig`, `*.rej`, `*.swp`, `*.swo`, `*.bak`, `*.tmp`, `*~`, `.DS_Store`, `Thumbs.db`), which docs/pregate.md and docs/troubleshooting.md now list too. +- **`literals`:** a named literal the branch only adds, such as `limit: 300` in a new object, was read as an existing `limit: 10` elsewhere in the file changing to 300, and every test that mentioned 10 was flagged. The check now compares every value under a name, so an added literal isn't a change. Editing an existing value still fails, and so do two cases the old last-value check could miss: changing one of two equal values under the same name, and deleting one site while editing another. + +### Tests +- `test/pregate-selection.test.ts`: `probe.py` and `temporal-*.test.ts` pass `strays` while `zz-notes.md`, `tmp-x.ts`, `scratch.py` and `util.js.orig` still fail; a new object literal beside `limit: 10` passes `literals` while editing that value still fails; the pure `changedLiterals` case keeps an exported constant change. + ## [1.94.22.0] - 2026-11-03 **/investigate, /review, /qa and /verify now say when to use them, and /investigate and /review declare what "done" means (GST-1088 lane B).** Untold, Opus 5.5 never opened /investigate in 252 trials (Run D, gstack-evals#140). A when-to-use description raised GPT-6.1 Sol's open rate from 9.5% to 27.0% in the same run. Claude Code's catalog shows only a description's first sentence, so each of the four now opens with a "Use when ..." sentence. The routing hook and the contract check that read the new frontmatter landed in v1.94.6.0 (lane A, off by default; see docs/skill-routing-and-contracts.md), so the "Done means" list now also names `gstack-skill-contract check` and `waive`. diff --git a/VERSION b/VERSION index 23b0a03fb..7705a204f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.22.0 +1.94.23.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index d442974f0..c548bf908 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.22.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.23.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/pregate.md b/docs/pregate.md index f0f46da40..66771f548 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -201,9 +201,12 @@ Two stages with separate clocks, both printed: file is read whole from disk. A file over the size limit, or whose full diff does not parse, is scanned on its added lines alone, without that context. - `strays`: a new file at the repository root, a new top-level directory, or a - scratch name (`zz-*`, `tmp*`, `scratch*`) anywhere. + scratch name (`zz-*`, `tmp*`, `scratch*`) or editor leftover (`*.orig`, + `*.rej`, `*.swp`, `*.swo`, `*.bak`, `*.tmp`, `*~`, `.DS_Store`, `Thumbs.db`) + anywhere. - `literals`: for every changed exported constant or named timeout/retry - literal, the whole test tree (nightly- and Heavy-only files included) is + literal (a named literal the branch only adds, such as `limit: 300` in a new + object, is not a change), the whole test tree (nightly- and Heavy-only files included) is searched for the old value in files that name the constant or, when the old value is longer than one character, its module (the module's file name not followed by another extension, or a path ending diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f17f55959..182296097 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3789,7 +3789,9 @@ anchored pattern and a reason ([docs/pregate.md](pregate.md#allowlist)); ### `PREGATE_STRAYS` **Meaning.** A file was added at the repository root, in a new top-level -directory, or under a scratch name (`zz-*`, `tmp*`, `scratch*`). +directory, or under a scratch name (`zz-*`, `tmp*`, `scratch*`) or an editor +leftover name (`*.orig`, `*.rej`, `*.swp`, `*.swo`, `*.bak`, `*.tmp`, `*~`, +`.DS_Store`, `Thumbs.db`). **Fix.** Delete or move it. An intentional root file goes in `strays.allow`. diff --git a/lib/pregate/checks/literals.ts b/lib/pregate/checks/literals.ts index b34311e09..a1db40bb7 100644 --- a/lib/pregate/checks/literals.ts +++ b/lib/pregate/checks/literals.ts @@ -57,10 +57,10 @@ export function constantsIn(text: string): Map { return out; } -/** Named numeric call sites (`timeout: 5000`) keyed by name; the last value per name wins, which is enough to spot a change. */ -export function callLiteralsIn(text: string): Map { - const out = new Map(); - for (const m of text.matchAll(CALL_PATTERN)) out.set(m[1]!, m[2]!); +/** Named numeric call sites (`timeout: 5000`) keyed by name, every value in source order. */ +export function callLiteralsIn(text: string): Map { + const out = new Map(); + for (const m of text.matchAll(CALL_PATTERN)) out.set(m[1]!, [...(out.get(m[1]!) ?? []), m[2]!]); return out; } @@ -69,7 +69,14 @@ export function changedLiterals(file: string, oldText: string, newText: string): const oldC = constantsIn(oldText); const newC = constantsIn(newText); for (const [name, old] of oldC) { const now = newC.get(name); if (now !== undefined && now !== old) out.push({ name, old, new: now, file }); } const oldCall = callLiteralsIn(oldText); const newCall = callLiteralsIn(newText); - for (const [name, old] of oldCall) { const now = newCall.get(name); if (now !== undefined && now !== old && !oldC.has(name)) out.push({ name, old, new: now, file }); } + for (const [name, olds] of oldCall) { + if (oldC.has(name)) continue; + const added = [...(newCall.get(name) ?? [])]; + const removed = olds.filter(v => { const i = added.indexOf(v); if (i >= 0) added.splice(i, 1); return i < 0; }); + if (!added.length) continue; + const seen = new Set(); + removed.forEach((old, i) => { if (!seen.has(old)) { seen.add(old); out.push({ name, old, new: added[Math.min(i, added.length - 1)]!, file }); } }); + } return out; } diff --git a/lib/pregate/checks/strays.ts b/lib/pregate/checks/strays.ts index 4b5d5b572..fed27ae1d 100644 --- a/lib/pregate/checks/strays.ts +++ b/lib/pregate/checks/strays.ts @@ -12,7 +12,7 @@ import type { CheckDef, CheckResult, PregateContext } from '../types'; /** Conventional root files a repo adds without a declaration. */ export const ROOT_OK = /^(?:README|LICENSE|LICENCE|NOTICE|CHANGELOG|CONTRIBUTING|CODE_OF_CONDUCT|SECURITY|AUTHORS|VERSION|Makefile|Dockerfile|Procfile|Gemfile|Rakefile|Cargo\.(?:toml|lock)|go\.(?:mod|sum)|package(?:-lock)?\.json|bun\.lockb?|pnpm-lock\.yaml|yarn\.lock|tsconfig[\w.-]*\.json|bunfig\.toml|pyproject\.toml|pubspec\.(?:yaml|lock)|analysis_options\.yaml|dart_test\.yaml|requirements[\w.-]*\.txt|setup\.(?:py|cfg)|\.[\w.-]+|[\w.-]+\.config\.[cm]?[jt]s|[\w.-]+\.md)$/i; -export const STRAY_NAME = /^(?:zz-|tmp|temp|scratch|debug-|probe)|\.(?:orig|rej|swp|swo|bak|tmp)$|~$|^\.DS_Store$|^Thumbs\.db$/i; +export const STRAY_NAME = /^(?:zz-|tmp|scratch)|\.(?:orig|rej|swp|swo|bak|tmp)$|~$|^\.DS_Store$|^Thumbs\.db$/i; function run(ctx: PregateContext): CheckResult { const inputs = baseInputs(ctx); diff --git a/package.json b/package.json index 672f1eebb..f9642ffac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.22", + "version": "1.94.23", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/pregate-selection.test.ts b/test/pregate-selection.test.ts index 2069493f7..d0f9e7a02 100644 --- a/test/pregate-selection.test.ts +++ b/test/pregate-selection.test.ts @@ -201,6 +201,16 @@ describe('literals (pure)', () => { { name: 'MAX', old: '3', new: '4', file: 'f.ts' }, { name: 'timeout', old: '5000', new: '9000', file: 'f.ts' }, ]); }); + + test('GST-1211: a key inside a newly added object literal is not a changed literal; an edited call-site value and an exported constant still are', () => { + const before = 'export const MAX = 3;\nconst page = { limit: 10 };\n'; + expect(changedLiterals('f.ts', before, `${before}const query = { selection: { order: 'asc', limit: 300 } };\n`)).toEqual([]); + expect(changedLiterals('f.ts', before, 'export const MAX = 3;\nconst page = { limit: 20 };\n')).toEqual([{ name: 'limit', old: '10', new: '20', file: 'f.ts' }]); + expect(changedLiterals('f.ts', before, 'export const MAX = 4;\nconst page = { limit: 10 };\nconst query = { limit: 300 };\n')).toEqual([{ name: 'MAX', old: '3', new: '4', file: 'f.ts' }]); + expect(changedLiterals('f.ts', 'const retired = { limit: 10 };\nconst page = { limit: 20 };\n', 'const page = { limit: 30 };\n')).toEqual([ + { name: 'limit', old: '10', new: '30', file: 'f.ts' }, { name: 'limit', old: '20', new: '30', file: 'f.ts' }, + ]); + }); // Value: protects=a sibling file that shares the module's stem (`postgres-unit-arms.txt` beside `postgres-unit-arms.ts`) is not the // module, so a test that only names it is not a consumer, while an extensionless import, a JS/TS import extension, the module's own // file name and a path ending at the stem (then a quote, space, bracket, sentence end or `.d.ts`) still are, and neither `.snap` nor a JS twin of a @@ -519,6 +529,21 @@ describe('gstack-pregate (bin)', () => { expect(w.out).toMatch(/^preflight {2}strays {4}warn .*downgraded to warn by origin\/main@/m); }); + test('GST-1211: probe.py and temporal-*.test.ts are not scratch names; zz-*, tmp*, scratch* and editor leftovers still are', () => { + const repo = fixture({ pregate: { ...BASE_PREGATE, warn: [] } }); + commit(repo, { 'docs/comparison-systems/ext-temporal-graph/shipped-0.31/probe.py': 'print(1)\n', 'test/eval/temporal-graph-shipped.test.ts': 'export {};\n' }); + const ok = pregate(repo, ['--stage', 'preflight', '--only', 'strays']); + expect(ok.status).toBe(EXIT.ok); + expect(ok.out).toMatch(/^preflight {2}strays {4}pass /m); + commit(repo, { 'docs/zz-notes.md': '# n\n', 'lib/tmp-x.ts': 'export {};\n', 'test/scratch.py': 'pass\n', 'lib/util.js.orig': 'x\n' }); + const r = pregate(repo, ['--stage', 'preflight', '--only', 'strays']); + expect(r.status).toBe(EXIT.fail); + expect(r.out).toMatch(/^preflight {2}strays {4}fail .*4 stray file\(s\)/m); + for (const f of ['docs/zz-notes.md', 'lib/tmp-x.ts', 'test/scratch.py', 'lib/util.js.orig']) expect(r.out).toContain(`strays: ${f}: scratch name (${f.split('/').pop()})`); + expect(r.out).not.toContain('probe.py: scratch name'); + expect(r.out).not.toContain('temporal-graph-shipped.test.ts: scratch name'); + }); + test('p0-4-heavy-old-constant: a nightly-only test still asserting the old literal fails literals with file:line', () => { const repo = fixture(); commit(repo, { 'lib/retry.js': "exports.MAX_RETRIES = 3;\nexports.TIMEOUT_MS = 9000;\nexports.retry = (fn) => fn();\n" }); @@ -532,6 +557,18 @@ describe('gstack-pregate (bin)', () => { expect(pregate(repo, ['--stage', 'preflight', '--only', 'literals']).status).toBe(EXIT.ok); }); + test('GST-1211: a new object literal beside an existing `limit: 10` passes literals; editing that existing value still fails', () => { + const repo = fixture({ baseFiles: { 'lib/query.js': 'exports.page = () => ({ limit: 10 });\n', 'test/query.test.js': "const test = require('node:test');\nconst assert = require('node:assert');\nconst { page } = require('../lib/query.js');\ntest('pages by ten', () => assert.strictEqual(page().limit, 10));\n" } }); + commit(repo, { 'lib/query.js': "exports.page = () => ({ limit: 10 });\nexports.selection = { order: 'asc', limit: 300 };\n" }); + const ok = pregate(repo, ['--stage', 'preflight', '--only', 'literals']); + expect(ok.status).toBe(EXIT.ok); + expect(ok.out).not.toContain('test/query.test.js'); + commit(repo, { 'lib/query.js': "exports.page = () => ({ limit: 20 });\nexports.selection = { order: 'asc', limit: 300 };\n" }); + const r = pregate(repo, ['--stage', 'preflight', '--only', 'literals']); + expect(r.status).toBe(EXIT.fail); + expect(r.out).toContain('literals: test/query.test.js:4: limit still 10 (now 20 in lib/query.js)'); + }); + // Value: protects=a real stale one-character constant still fails literals end to end after the #3492 narrowing, and an incidental // digit in the same file is not what it reports; fails_when=the statement binding drops a named assertion; why_new=only // multi-character values had a bin-level control; seam=none; no_claim=two-character values From 1ccab0d8bccba50e69587672fb82ac0574638a65 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 07:41:57 -0800 Subject: [PATCH 5/5] v1.94.24.0 fix(harness): start hook leaves a fresh harness config alone, keeps the profile's own OPENCLAW_CONFIG_PATH, reads status off PATH (GST-778) (#3794) * fix(harness): start hook waits for the harness config, keeps the profile's own OPENCLAW_CONFIG_PATH, reads status from config off PATH (GST-778) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): recovery note covers every release before this fix Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(hosts): turning the hook off in a profile with no harness config runs no harness command Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-harness-ledger.sh | 8 +- docs/hosts.md | 10 +- docs/troubleshooting.md | 6 +- lib/harness-register.ts | 20 ++-- lib/harness-start-hook.ts | 59 +++++++++++- package.json | 2 +- test/harness-start-hook.test.ts | 156 +++++++++++++++++++++++++++++++- 10 files changed, 246 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b148dc2..6f411c10c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.94.24.0] - 2026-11-03 + +### Fixed + +- **The OpenClaw and Hermes start hook no longer breaks a fresh harness.** When the harness has no config yet, gstack runs none of its commands: `openclaw config set` and `hermes plugins enable` would have created a config holding only gstack's plugin setting, with no model, provider or terminal settings. The hook reports `written` with the setup to run first (`openclaw onboard`, `hermes setup`), and the next `./setup` or `gstack-harness refresh` turns it on, changing only the plugin entry. Turning the hook off or unregistering in such a profile also leaves the config alone. If v1.92.23.0 or a later release before this fix already created a config holding only the plugin entry, move it aside, run the harness's own setup, then `gstack-harness refresh`. +- **`OPENCLAW_CONFIG_PATH` naming the profile's own `openclaw.json` keeps the start hook on.** gstack compares the path after resolving symlinks, so only a config outside the profile turns the hook off. When `GSTACK_HARNESS_START_HOOK` or `harness_start_hook` asked for the hook and it ends up off, `gstack-harness register` and `./setup --host` now exit 4 with the reason and the fix instead of exiting 0. +- **`gstack-harness status` reads the hook from the harness's config when the harness CLI is not on `PATH`**, as inside Hermes's or OpenClaw's own terminal tool. Hermes's `config.yaml` settles it (`active`, `disabled` or `written`); otherwise status says `unknown` instead of a wrong `written`. + ## [1.94.23.0] - 2026-11-03 **Two `gstack-pregate` preflight false positives fixed (GST-1211).** Both stopped /ship on real work in gbrain-evals. diff --git a/VERSION b/VERSION index 7705a204f..7a11af84d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.23.0 +1.94.24.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index c548bf908..c74798cc8 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.23.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.24.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-harness-ledger.sh b/bin/gstack-harness-ledger.sh index 6d24acb4b..540a9b977 100644 --- a/bin/gstack-harness-ledger.sh +++ b/bin/gstack-harness-ledger.sh @@ -74,9 +74,10 @@ _gstack_harness_cut_block() { # plugins.entries.gstack-start; Hermes removes the plugin and its config entries when its # folder holds exactly the files gstack wrote, unedited, and is reached without a symlink # below the profile; otherwise it only disables it so an edit or a kept copy survives. The harness runs with PATH, HOME and the profile only. Without the harness on -# PATH it prints the command to run instead. +# PATH it prints the command to run instead. A profile with no harness config (openclaw.json, config.yaml) never had +# the plugin turned on, and the harness CLI would create a config holding only that change, so it runs no harness command. gstack_harness_hook_release() { - local ledger="$1" kind a b target="" on="" profile="" root="" edited=0 to="" sub cmd n=0 safe=1 total + local ledger="$1" kind a b target="" on="" profile="" root="" edited=0 to="" sub cmd cfg n=0 safe=1 total [ -f "$ledger" ] || return 0 while IFS=$'\t' read -r kind a b; do case "$kind" in @@ -91,6 +92,8 @@ gstack_harness_hook_release() { [ "${total:-0}" -gt "$n" ] && edited=1 fi command -v timeout >/dev/null 2>&1 && to="timeout 60" + case "$target" in openclaw) cfg=openclaw.json ;; hermes) cfg=config.yaml ;; *) cfg="" ;; esac + if [ -z "$cfg" ] || [ -f "$profile/$cfg" ]; then case "$target" in openclaw) cmd="config unset plugins.entries.gstack-start" if command -v openclaw >/dev/null 2>&1 && env -i PATH="$PATH" HOME="$HOME" OPENCLAW_STATE_DIR="$profile" $to openclaw $cmd /dev/null 2>&1; then : @@ -99,6 +102,7 @@ gstack_harness_hook_release() { if command -v hermes >/dev/null 2>&1 && env -i PATH="$PATH" HOME="$HOME" HERMES_HOME="$profile" $to hermes plugins "$sub" gstack-start /dev/null 2>&1; then : else echo " start hook: turn it off in Hermes yourself: HERMES_HOME=\"$profile\" hermes plugins disable gstack-start"; fi ;; esac + fi if [ "$safe" = 1 ] && [ -d "$root/__pycache__" ] && _gstack_harness_owned "$root/__pycache__" "$root"; then rm -f "$root"/__pycache__/*.pyc rmdir "$root/__pycache__" 2>/dev/null diff --git a/docs/hosts.md b/docs/hosts.md index ecabfbf76..4c465c7f1 100644 --- a/docs/hosts.md +++ b/docs/hosts.md @@ -80,14 +80,16 @@ It is off unless you turn it on, because turning it on changes the harness's own |---|---|---| | Plugin folder | `${OPENCLAW_STATE_DIR:-~/.openclaw}/extensions/gstack-start` (index.js, openclaw.plugin.json, package.json, note.txt) | `${HERMES_HOME:-~/.hermes}/plugins/gstack-start` (plugin.yaml, __init__.py, note.txt) | | Hook | `before_prompt_build`, returning the note as `prependContext` | `pre_llm_call`, returning the note as `context` | -| Settings it changes, through the harness's own CLI | `openclaw config set plugins.entries.gstack-start.hooks.allowConversationAccess true` and `...enabled true` in `openclaw.json`. OpenClaw runs `before_prompt_build` from a plugin it did not bundle only with that grant, which also lets the plugin's hooks see conversation content (the plugin reads only its own note.txt) | `hermes plugins enable gstack-start` adds it to `plugins.enabled` in `config.yaml` (and creates `config.yaml` when there is none) | +| Settings it changes, through the harness's own CLI | `openclaw config set plugins.entries.gstack-start.hooks.allowConversationAccess true` and `...enabled true` in `openclaw.json`. OpenClaw runs `before_prompt_build` from a plugin it did not bundle only with that grant, which also lets the plugin's hooks see conversation content (the plugin reads only its own note.txt) | `hermes plugins enable gstack-start` adds it to `plugins.enabled` in `config.yaml` | | Check | `openclaw plugins inspect gstack-start --runtime --json` lists `before_prompt_build` under `typedHooks` | `hermes plugins list` shows `gstack-start` enabled | -**Turn it on.** `gstack-harness register openclaw --start-hook` (or `hermes`), or `gstack-config set harness_start_hook on` followed by `./setup`, or `GSTACK_HARNESS_START_HOOK=1 ./setup --host openclaw`. The first of these that is set decides: the `--start-hook`/`--no-start-hook` flag, then `GSTACK_HARNESS_START_HOOK` (`1`/`on` or `0`/`off`), then `harness_start_hook`, then the choice saved in the registration's ledger. Whatever decides, the choice is saved, so refresh and upgrades keep it. The hook goes only into the harness profile's own plugin folder; a registration with a custom `--skills-dir`, or OpenClaw run with `OPENCLAW_CONFIG_PATH` pointing at another config, gets no hook (`HARNESS_HOOK_UNSUPPORTED` when you ask for one). A hook you already turned on stays on when a later run happens to have `OPENCLAW_CONFIG_PATH` set. +**Turn it on.** `gstack-harness register openclaw --start-hook` (or `hermes`), or `gstack-config set harness_start_hook on` followed by `./setup`, or `GSTACK_HARNESS_START_HOOK=1 ./setup --host openclaw`. The first of these that is set decides: the `--start-hook`/`--no-start-hook` flag, then `GSTACK_HARNESS_START_HOOK` (`1`/`on` or `0`/`off`), then `harness_start_hook`, then the choice saved in the registration's ledger. Whatever decides, the choice is saved, so refresh and upgrades keep it. The hook goes only into the harness profile's own plugin folder; a registration with a custom `--skills-dir`, or OpenClaw run with `OPENCLAW_CONFIG_PATH` pointing at another config, gets no hook (`HARNESS_HOOK_UNSUPPORTED` with `--start-hook`; with `GSTACK_HARNESS_START_HOOK` or `harness_start_hook`, `register` and `./setup --host` say why and exit 4). `OPENCLAW_CONFIG_PATH` naming the profile's own `openclaw.json`, by any path that resolves to it, is fine. A hook you already turned on stays on when a later run happens to have `OPENCLAW_CONFIG_PATH` set. -**Check it.** The receipt and `gstack-harness status` show `start hook: active` (configured for new sessions; a running OpenClaw gateway or Hermes gateway picks it up when it reloads or restarts), `written` (files in place but not active, with the reason and the command that fixes it), `disabled` (you turned it off in the harness), `unchecked` (a run that left a hook already on alone, or `status --no-check`; `gstack-harness status` checks it) or `off`. When the run that turns the hook on (or `--start-hook`) leaves it inactive, `gstack-harness register` and `./setup --host` exit 4 after registering the skills (`HARNESS_HOOK_INACTIVE`). Later refreshes and upgrades try once more each time, without failing, until the hook is active or you disable the plugin in the harness. +**Run the harness's own setup first.** gstack turns the hook on only in a harness that already has its config (`openclaw.json` from `openclaw onboard`, `config.yaml` from `hermes setup`). Without one, the harness CLI would create a config holding nothing but gstack's plugin setting, with no model, provider or terminal settings, so gstack runs no harness command: the plugin's files are written, the hook reports `written` with the setup to run, and `register` and `./setup --host` exit 4. Once the harness is set up, the next `./setup` or `gstack-harness refresh` turns the hook on, changing only the `gstack-start` plugin entry. -**Turn it off.** `gstack-harness register openclaw --no-start-hook` (or `hermes`), or `gstack-config set harness_start_hook off` followed by `./setup`. Either turns the plugin off through the harness's CLI and removes its folder. You can also switch it off in the harness (`openclaw config set plugins.entries.gstack-start.enabled false`, `hermes plugins disable gstack-start`); gstack then leaves it off and never turns it back on unless you pass `--start-hook` again. `gstack-harness unregister` and `gstack-uninstall` remove it with the rest (Hermes only disables it when its folder holds an edited file or a kept `.edited-*` copy, so your edit survives), and print the harness command to run when the harness CLI is not on `PATH`. +**Check it.** The receipt and `gstack-harness status` show `start hook: active` (configured for new sessions; a running OpenClaw gateway or Hermes gateway picks it up when it reloads or restarts), `written` (files in place but not active, with the reason and the command that fixes it), `disabled` (you turned it off in the harness), `unknown` (the harness CLI is not on `PATH`, as inside OpenClaw's or Hermes's own terminal tool, and the config file cannot settle it; gstack reads `plugins.enabled` and `plugins.disabled` from Hermes's `config.yaml` and the plugin entry from `openclaw.json` instead), `unchecked` (a run that left a hook already on alone, or `status --no-check`; `gstack-harness status` checks it) or `off`. When the run that turns the hook on (or `--start-hook`) leaves it inactive, `gstack-harness register` and `./setup --host` exit 4 after registering the skills (`HARNESS_HOOK_INACTIVE`). Later refreshes and upgrades try once more each time, without failing, until the hook is active or you disable the plugin in the harness. + +**Turn it off.** `gstack-harness register openclaw --no-start-hook` (or `hermes`), or `gstack-config set harness_start_hook off` followed by `./setup`. Either turns the plugin off through the harness's CLI and removes its folder. You can also switch it off in the harness (`openclaw config set plugins.entries.gstack-start.enabled false`, `hermes plugins disable gstack-start`); gstack then leaves it off and never turns it back on unless you pass `--start-hook` again. `gstack-harness unregister` and `gstack-uninstall` remove it with the rest (Hermes only disables it when its folder holds an edited file or a kept `.edited-*` copy, so your edit survives), and print the harness command to run when the harness CLI is not on `PATH`. In a profile with no harness config yet (no `openclaw.json` or `config.yaml`), the hook was never turned on, so turning it off or unregistering runs no harness command and leaves the config alone. **What it costs.** The note is about 50 tokens and lands on every user turn, and each copy stays in the session's history, so a long session pays more (mostly cache reads). HARN-18's panel measures the added cost per trial. OpenClaw also logs that it cannot verify where the plugin came from, because gstack writes it instead of installing it from npm or ClawHub. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 182296097..c7854d943 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -6472,7 +6472,7 @@ There is no ledger for that harness or id under `$GSTACK_STATE_ROOT/harness`. `g ### `gstack-harness: has no start hook` / `the start hook goes into the profile , and this registration uses another skills folder` (HARNESS_HOOK_UNSUPPORTED) -The [start hook](hosts.md#start-hook) exists for OpenClaw and Hermes only, and it goes into the harness profile's own plugin folder, the one the harness scans. A registration with `--skills-dir` somewhere else has no profile gstack can trust, and with `OPENCLAW_CONFIG_PATH` set OpenClaw reads a config outside the profile, where gstack's settings change would not land, so `--start-hook` is refused and nothing is written. Register without `--skills-dir` (set `OPENCLAW_STATE_DIR` or `HERMES_HOME` to pick the profile), or drop `--start-hook`. When the choice comes from `GSTACK_HARNESS_START_HOOK` or `harness_start_hook` instead of the flag, such a registration just gets no hook, and its receipt says so. +The [start hook](hosts.md#start-hook) exists for OpenClaw and Hermes only, and it goes into the harness profile's own plugin folder, the one the harness scans. A registration with `--skills-dir` somewhere else has no profile gstack can trust, and with `OPENCLAW_CONFIG_PATH` naming another config OpenClaw reads settings outside the profile, where gstack's settings change would not land, so `--start-hook` is refused and nothing is written. `OPENCLAW_CONFIG_PATH` naming the profile's own `openclaw.json` (compared after resolving symlinks) is not refused. Register without `--skills-dir` (set `OPENCLAW_STATE_DIR` or `HERMES_HOME` to pick the profile), unset `OPENCLAW_CONFIG_PATH` or point it at the profile's own `openclaw.json`, or drop `--start-hook`. When the choice comes from `GSTACK_HARNESS_START_HOOK` or `harness_start_hook` instead of the flag, the registration gets no hook, and `gstack-harness register` and `./setup --host` exit 4 with the reason (`HARNESS_HOOK_INACTIVE`, `start hook is off`); refreshes only print it. ### `gstack-harness: GSTACK_HARNESS_START_HOOK="" is not on or off` (HARNESS_HOOK_CHOICE_INVALID) @@ -6487,7 +6487,9 @@ Each harness profile has one start hook, owned by the gstack registration that w ### `gstack-harness: start hook is written: . Turn it on: ` (HARNESS_HOOK_INACTIVE, exit 4) -The skills are registered and the start hook's files are in place, but the hook is not active in the harness. gstack turns the hook on through the harness's own CLI and then reads it back, so this means the CLI was not on `PATH`, one of its commands failed or timed out (the message quotes its last output), or the harness's settings block the plugin: OpenClaw's `plugins.allow` set without `gstack-start`, `gstack-start` in `plugins.deny`, or `hooks.allowPromptInjection: false`. gstack never edits your allow or deny lists. Run the printed command, then `gstack-harness status`; `--start-hook` again retries the whole step. `./setup --host` exits 4 for the same reason. Refreshes and upgrades retry on their own without failing, and stop once you disable the plugin in the harness. +The skills are registered and the start hook's files are in place, but the hook is not active in the harness. gstack turns the hook on through the harness's own CLI and then reads it back, so this means the harness has no config yet (`no OpenClaw config at yet` or `no Hermes config at yet`: run `openclaw onboard` or `hermes setup` first, then `gstack-harness refresh`; gstack runs no harness command until then, because the CLI would create a config holding only gstack's plugin setting), the CLI was not on `PATH`, one of its commands failed or timed out (the message quotes its last output), or the harness's settings block the plugin: OpenClaw's `plugins.allow` set without `gstack-start`, `gstack-start` in `plugins.deny`, or `hooks.allowPromptInjection: false`. gstack never edits your allow or deny lists. Run the printed command, then `gstack-harness status`; `--start-hook` again retries the whole step. `./setup --host` exits 4 for the same reason. Refreshes and upgrades retry on their own without failing, and stop once you disable the plugin in the harness. + +The same code with `start hook is off: ` means you asked for the hook through `GSTACK_HARNESS_START_HOOK` or `harness_start_hook` but this registration cannot take it (a custom `--skills-dir`, `OPENCLAW_CONFIG_PATH` naming another config, or an unsafe plugin folder); the message names the fix. ### `gstack-harness: no QM admin credential in the environment` / `... needs --yes` (HARNESS_QM_CREDENTIALS) diff --git a/lib/harness-register.ts b/lib/harness-register.ts index 9b00b665f..8744f0977 100644 --- a/lib/harness-register.ts +++ b/lib/harness-register.ts @@ -22,7 +22,7 @@ import { spawnSync } from 'node:child_process'; import { bunIsolationArgs } from './bun-isolation'; import { runBashSync } from './script-invocation'; import { frontmatterList, routerLine } from './skill-triggers'; -import { activateHook, assertHookRoot, hookChoice, hookFiles, hookProfile, hookRoot, inspectHook, isHookTarget, profileSkills, type HookResult } from './harness-start-hook'; +import { activateHook, assertHookRoot, foreignOpenclawConfig, hookChoice, hookConfig, hookFiles, hookProfile, hookRoot, inspectHook, isHookTarget, profileSkills, type HookResult } from './harness-start-hook'; export type SkillsMode = 'router' | 'hybrid' | 'all'; export type TargetName = 'openclaw' | 'hermes' | 'dots' | 'generic' | 'qm' | 'muse' | 'grok'; @@ -135,7 +135,7 @@ export const HARNESS_CODES = { HARNESS_RENDER_FAILED: { anchor: 'harness-render-failed', fix: 'run bun install in the gstack checkout, then register again; the render output above names the failing skill' }, HARNESS_BUDGET_EXCEEDED: { anchor: 'harness-budget-exceeded', fix: 'this is a gstack bug: report it with the byte count printed above' }, HARNESS_NOT_REGISTERED: { anchor: 'harness-not-registered', fix: 'run gstack-harness status to see the registrations on this machine' }, - HARNESS_HOOK_UNSUPPORTED: { anchor: 'harness-hook-unsupported', fix: 'the start hook is for OpenClaw and Hermes registered in their own profile: drop --skills-dir (set OPENCLAW_STATE_DIR or HERMES_HOME to pick the profile), or drop --start-hook' }, + HARNESS_HOOK_UNSUPPORTED: { anchor: 'harness-hook-unsupported', fix: 'the start hook is for OpenClaw and Hermes registered in their own profile: drop --skills-dir (set OPENCLAW_STATE_DIR or HERMES_HOME to pick the profile), unset OPENCLAW_CONFIG_PATH or point it at the profile\'s own openclaw.json, or drop --start-hook' }, HARNESS_HOOK_CHOICE_INVALID: { anchor: 'harness-hook-choice-invalid', fix: 'set it to on or off (GSTACK_HARNESS_START_HOOK also takes 1 or 0), or unset it' }, HARNESS_HOOK_OWNED_ELSEWHERE: { anchor: 'harness-hook-owned-elsewhere', fix: 'if another gstack install wrote it, run that install\'s gstack-harness register --no-start-hook; otherwise move the folder aside, then register again' }, HARNESS_HOOK_INACTIVE: { anchor: 'harness-hook-inactive', fix: 'run the command printed above, then gstack-harness status; the skills themselves are registered' }, @@ -525,7 +525,7 @@ function shellRegistry(opts: RegisterOptions, fn: string, args: string[]): boole return sourcedBash(opts.checkout, 'gstack-install-registry.sh', fn, args, opts.stateRoot, opts.env).status === 0; } -interface HookPlan { on: boolean; prevOn: boolean; explicit: boolean; retry: boolean; profile: string; root: string; files: Map; note?: string } +interface HookPlan { on: boolean; prevOn: boolean; explicit: boolean; retry: boolean; profile: string; root: string; files: Map; note?: string; fix?: string; requested?: boolean } /** The start hook for this run: whether it is on, where it goes, and its files (null for targets without a hook). */ function hookPlan(opts: RegisterOptions, dest: string, prev: Ledger | null): HookPlan | null { @@ -544,17 +544,19 @@ function hookPlan(opts: RegisterOptions, dest: string, prev: Ledger | null): Hoo const name = t.name; const off = (p: HookPlan) => { if (saved) assertHookRoot(name, saved.profile, saved.root, saved.root, (code, msg) => { throw new HarnessError(code, msg); }); return p; }; if (!choice.on) return off(plan); - const why = dest !== profileSkills(t.name, profile) ? 'this registration uses another skills folder (' + dest + ')' - : t.name === 'openclaw' && opts.env.OPENCLAW_CONFIG_PATH ? 'OPENCLAW_CONFIG_PATH points OpenClaw at a config outside that profile' : ''; + const requested = !opts.inherit; + const foreign = t.name === 'openclaw' ? foreignOpenclawConfig(opts.env, profile) : ''; + const [why, fix] = dest !== profileSkills(t.name, profile) ? ['this registration uses another skills folder (' + dest + ')', 'register without --skills-dir to get the hook, or pass --no-start-hook to keep this registration without it'] + : foreign ? ['OPENCLAW_CONFIG_PATH points OpenClaw at ' + foreign + ', outside that profile', 'unset OPENCLAW_CONFIG_PATH or set it to ' + hookConfig('openclaw', profile) + ', then register again'] : ['', '']; if (why && (plan.explicit || !prevOn)) { if (plan.explicit) throw new HarnessError('HARNESS_HOOK_UNSUPPORTED', 'the start hook goes into the ' + t.displayName + ' profile ' + profile + ', and ' + why); - return off({ ...plan, on: false, note: 'off: ' + why + ' (' + choice.source + ' asked for the hook)' }); + return off({ ...plan, on: false, requested, fix, note: 'off: ' + why + ' (' + choice.source + ' asked for the hook)' }); } try { assertHookRoot(t.name, profile, root, prevOn ? prev!.meta.hook_root : undefined, (code, msg) => { throw new HarnessError(code, msg); }); } catch (e) { if (plan.explicit || prevOn || !(e instanceof HarnessError)) throw e; - return off({ ...plan, on: false, note: 'off: ' + e.message + ' (' + choice.source + ' asked for the hook; ' + HARNESS_CODES[e.code].fix + ')' }); + return off({ ...plan, on: false, requested, fix: HARNESS_CODES[e.code].fix, note: 'off: ' + e.message + ' (' + choice.source + ' asked for the hook)' }); } return { ...plan, files: hookFiles(t.name, root) }; } @@ -562,7 +564,7 @@ function hookPlan(opts: RegisterOptions, dest: string, prev: Ledger | null): Hoo /** Turns the hook on after its files are written (when this run turns it on, was asked to, or the last try did not end active), and reads it back. */ function finishHook(opts: RegisterOptions, hook: HookPlan | null, released: string): RegisterResult['hook'] { if (!hook) return undefined; - if (!hook.on) return { state: 'off', ...(hook.note ? { detail: hook.note } : {}), ...(released ? { fix: released } : {}) }; + if (!hook.on) return { state: 'off', ...(hook.note ? { detail: hook.note } : {}), ...(released || hook.fix ? { fix: released || hook.fix } : {}), ...(hook.requested ? { required: true } : {}) }; const t = opts.target.name as 'openclaw' | 'hermes'; const required = !hook.prevOn || hook.explicit; if (!required && !hook.retry) return { state: 'unchecked', profile: hook.profile, root: hook.root, fix: 'gstack-harness status shows whether it is active' }; @@ -678,7 +680,7 @@ export function register(opts: RegisterOptions): RegisterResult { if (opts.registry && t.render === 'host') shellRegistry(opts, 'gstack_install_registry_upsert', [t.name, 'global', '-', dest, opts.checkout, opts.checkout, opts.version, '-', '-']); if (same && hook?.on && hook.explicit && prev.meta.hook_active !== 'no') writeLedger(lfile, { ...prev, meta: { ...prev.meta, hook_active: 'no' } }); const hookResult = finishHook(opts, hook, released); - const active = hookResult?.required || hookResult?.retried ? ({ active: 'yes', disabled: 'disabled' } as Record)[hookResult.state] ?? 'no' : ''; + const active = hook?.on && (hookResult?.required || hookResult?.retried) ? ({ active: 'yes', disabled: 'disabled' } as Record)[hookResult!.state] ?? 'no' : ''; const saved = active ? readLedger(lfile) : null; if (saved && saved.meta.hook_active !== active) writeLedger(lfile, { ...saved, meta: { ...saved.meta, hook_active: active } }); return { id, target: t.name, dest, mode, status: !prev ? 'installed' : same ? 'unchanged' : 'updated', written, skills, router, ledger: lfile, kept, ...(hookResult ? { hook: hookResult } : {}) }; diff --git a/lib/harness-start-hook.ts b/lib/harness-start-hook.ts index 7594d6865..7529d0fd2 100644 --- a/lib/harness-start-hook.ts +++ b/lib/harness-start-hook.ts @@ -22,14 +22,14 @@ export const START_NOTE_MAX = 300; export const START_NOTE = 'gstack: if you use a gstack skill this turn, first run the gstack-skill-start command in its Preamble section exactly as written, with no pipe, head, || true or ; echo. Read its output, then follow the skill.'; export type HookTarget = 'openclaw' | 'hermes'; -export type HookState = 'active' | 'written' | 'disabled' | 'unchecked' | 'off'; +export type HookState = 'active' | 'written' | 'disabled' | 'unknown' | 'unchecked' | 'off'; export interface HookResult { state: HookState; fix?: string; detail?: string; profile?: string; root?: string } type Env = Record; const MANAGED = "Managed by gstack-harness: it rewrites this file. Turn the hook off: gstack-harness register %s --no-start-hook"; const SPECS = { openclaw: { - profileVar: 'OPENCLAW_STATE_DIR', home: '.openclaw', sub: 'extensions', skills: ['skills'], + profileVar: 'OPENCLAW_STATE_DIR', home: '.openclaw', sub: 'extensions', skills: ['skills'], name: 'OpenClaw', config: 'openclaw.json', setup: 'openclaw onboard', activate: [['config', 'set', 'plugins.entries.' + HOOK_ID + '.hooks.allowConversationAccess', 'true'], ['config', 'set', 'plugins.entries.' + HOOK_ID + '.enabled', 'true']], files: (): Record => ({ 'index.js': [ @@ -55,7 +55,7 @@ const SPECS = { }), }, hermes: { - profileVar: 'HERMES_HOME', home: '.hermes', sub: 'plugins', skills: ['skills', 'gstack'], + profileVar: 'HERMES_HOME', home: '.hermes', sub: 'plugins', skills: ['skills', 'gstack'], name: 'Hermes', config: 'config.yaml', setup: 'hermes setup', activate: [['plugins', 'enable', HOOK_ID]], files: (): Record => ({ 'plugin.yaml': ['# ' + MANAGED.replace('%s', 'hermes'), 'name: ' + HOOK_ID, 'version: 1.0.0', "description: Puts gstack's start-command note on each user turn (managed by gstack-harness).", ''].join('\n'), @@ -91,6 +91,51 @@ export const hookRoot = (t: HookTarget, profile: string) => path.join(profile, S /** The skills folder a registration must use for the hook to go into that profile. */ export const profileSkills = (t: HookTarget, profile: string) => path.join(profile, ...SPECS[t].skills); const activateCommands = (t: HookTarget, profile: string) => SPECS[t].activate.map(a => SPECS[t].profileVar + '=' + JSON.stringify(profile) + ' ' + t + ' ' + a.join(' ')).join(' && '); +/** The harness's own config file in a profile: the file activation edits through the harness CLI. */ +export const hookConfig = (t: HookTarget, profile: string) => path.join(profile, SPECS[t].config); + +function canonical(p: string): string { + try { return fs.realpathSync(p); } catch {} + const parent = path.dirname(p); + return parent === p ? p : path.join(canonical(parent), path.basename(p)); +} +/** The config OPENCLAW_CONFIG_PATH names when it is not the profile's own openclaw.json (compared after realpath), else ''. */ +export function foreignOpenclawConfig(env: Env, profile: string): string { + const v = env.OPENCLAW_CONFIG_PATH; + if (!v) return ''; + const named = path.resolve(v); + return canonical(named) === canonical(hookConfig('openclaw', profile)) ? '' : named; +} + +/** A profile with no harness config yet: turning the plugin on through the harness CLI would create a config holding only that setting, so gstack waits for the harness's own setup. */ +function noConfig(t: HookTarget, profile: string): HookResult | null { + const cfg = hookConfig(t, profile); + if (fs.existsSync(cfg)) return null; + const s = SPECS[t]; + return { state: 'written', detail: 'no ' + s.name + ' config at ' + cfg + ' yet, and turning the plugin on would create one holding only that setting', fix: 'run ' + s.name + "'s own setup first (" + s.profileVar + '=' + JSON.stringify(profile) + ' ' + s.setup + '), then gstack-harness refresh' }; +} + +/** Reads the hook from the harness's config file when its CLI is not on PATH (a harness's own terminal tool often lacks it). */ +function inspectConfig(t: HookTarget, profile: string, fix: string): HookResult { + const why = t + ' is not on PATH'; + const written: HookResult = { state: 'written', detail: why + ', so gstack could not turn the plugin on or check it', fix }; + const cfg = hookConfig(t, profile); + const missing = noConfig(t, profile); + if (missing) return { ...missing, detail: why + ' and ' + missing.detail }; + let plugins: any; + try { plugins = ((t === 'hermes' ? Bun.YAML.parse(fs.readFileSync(cfg, 'utf8')) : JSON.parse(fs.readFileSync(cfg, 'utf8'))) as any)?.plugins ?? {}; } + catch { return { state: 'unknown', detail: why + ' and ' + cfg + ' could not be parsed, so gstack could not check the plugin', fix: 'gstack-harness status from a shell where ' + t + ' is on PATH' }; } + if (t === 'hermes') { + const listed = (k: string) => Array.isArray(plugins[k]) && plugins[k].includes(HOOK_ID); + if (listed('disabled')) return { state: 'disabled', detail: 'turned off in Hermes (plugins.disabled in ' + cfg + '); gstack leaves it off', fix }; + if (listed('enabled') && fs.existsSync(path.join(hookRoot(t, profile), 'plugin.yaml'))) return { state: 'active', detail: 'read from plugins.enabled in ' + cfg + ' (' + why + ')' }; + return written; + } + const entry = plugins.entries?.[HOOK_ID] ?? {}; + if (entry.enabled === false) return { state: 'disabled', detail: 'turned off in openclaw.json (plugins.entries.' + HOOK_ID + '.enabled false); gstack leaves it off', fix }; + if (entry.enabled === true && entry.hooks?.allowConversationAccess === true) return { state: 'unknown', detail: cfg + ' turns it on, but ' + why + ', so gstack could not check that OpenClaw loads it', fix: 'gstack-harness status from a shell where openclaw is on PATH' }; + return written; +} /** The plugin's files (absolute path -> bytes): fixed code plus note.txt, which the code reads at each turn. */ export function hookFiles(t: HookTarget, root: string): Map { @@ -124,6 +169,8 @@ const onPath = (bin: string, env: Env) => (env.PATH ?? '').split(path.delimiter) /** Turns the plugin on in the harness. Returns the first failure, or ''. */ export function activateHook(t: HookTarget, profile: string, env: Env, home: string): string { if (!onPath(t, env)) return t + ' is not on PATH'; + const missing = noConfig(t, profile); + if (missing) return missing.detail!; for (const a of SPECS[t].activate) { const r = runHarness(t, profile, a, env, home, 60_000); if (!r.ok) return t + ' ' + a.join(' ') + ' failed (' + r.why + ')'; @@ -136,12 +183,14 @@ function parseJson(s: string): any { return null; } -/** Reads the hook back from the harness: active (configured for new sessions), disabled by the user, or written (with the fix). */ +/** Reads the hook back from the harness: active (configured for new sessions), disabled by the user, written (with the fix), or unknown (no CLI and a config that cannot settle it). Runs no harness command while the profile has no harness config. */ const READ_TIMEOUT = 15_000; export function inspectHook(t: HookTarget, profile: string, env: Env, home: string): HookResult { const fix = activateCommands(t, profile); - if (!onPath(t, env)) return { state: 'written', detail: t + ' is not on PATH, so gstack could not turn the plugin on or check it', fix }; + if (!onPath(t, env)) return inspectConfig(t, profile, fix); + const missing = noConfig(t, profile); + if (missing) return missing; if (t === 'hermes') { const r = runHarness(t, profile, ['plugins', 'list', '--json'], env, home, READ_TIMEOUT); const row = (parseJson(r.stdout) ?? []).find?.((p: any) => p?.name === HOOK_ID); diff --git a/package.json b/package.json index f9642ffac..187bdcb1f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.23", + "version": "1.94.24", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/harness-start-hook.test.ts b/test/harness-start-hook.test.ts index a226c2c6c..2564876a6 100644 --- a/test/harness-start-hook.test.ts +++ b/test/harness-start-hook.test.ts @@ -38,6 +38,13 @@ case "$1 $2" in hooks=""; [ -d "$d/extensions/gstack-start" ] && [ "$acc" = true ] && [ "$en" != false ] && hooks='{"name":"before_prompt_build"}' printf '{"plugin":{"status":"loaded","enabled":%s},"typedHooks":[%s]}\n' "$([ "$en" = false ] && echo false || echo true)" "$hooks" ;; esac +case "$1 $2" in "config set"|"config unset") + j="$d/openclaw.json"; b="$d/fake-base.json" + [ -f "$b" ] || { if [ -f "$j" ]; then cp "$j" "$b"; else printf '{"meta":{"lastTouchedVersion":"fake"}}' > "$b"; fi; } + acc=false; grep -q '^plugins.entries.gstack-start.hooks.allowConversationAccess=true$' "$c" && acc=true + en=$(grep '^plugins.entries.gstack-start.enabled=' "$c" | tail -1 | cut -d= -f2); enj=null; [ -n "$en" ] && enj=$en + if grep -q '^plugins.entries.gstack-start' "$c"; then printf '%s,"plugins":{"entries":{"gstack-start":{"enabled":%s,"hooks":{"allowConversationAccess":%s}}}}}' "$(sed 's/}$//' "$b")" "$enj" "$acc" > "$j"; else cp "$b" "$j"; fi ;; +esac `; const FAKE_HERMES = String.raw`#!/bin/sh d="$HERMES_HOME" @@ -48,10 +55,19 @@ case "$1 $2" in "plugins remove") rm -rf "$d/plugins/$3" "$d/fake-state" ;; "plugins list") if [ -d "$d/plugins/gstack-start" ]; then printf '[{"name":"gstack-start","status":"%s"}]\n' "$(cat "$d/fake-state" 2>/dev/null || echo 'not enabled')"; else echo '[]'; fi ;; esac +case "$1 $2" in "plugins enable"|"plugins disable"|"plugins remove") + y="$d/config.yaml"; b="$d/fake-base.yaml" + [ -f "$b" ] || { if [ -f "$y" ]; then cp "$y" "$b"; else : > "$b"; fi; } + s=$(cat "$d/fake-state" 2>/dev/null) + { cat "$b"; [ -z "$s" ] || printf 'plugins:\n %s:\n - gstack-start\n' "$s"; } > "$y.t"; mv "$y.t" "$y" ;; +esac `; +/** A configured harness: what each harness's own setup leaves (OpenClaw onboarding, hermes setup) before gstack runs. */ +const HEALTHY = { openclaw: '{"agents":{"defaults":{"model":{"primary":"anthropic/claude-opus-5-5"}}},"gateway":{"mode":"local"}}', hermes: 'model:\n default: claude-opus-5-5\n provider: anthropic\nterminal:\n backend: local\n' }; +const CONFIG = { openclaw: 'openclaw.json', hermes: 'config.yaml' }; /** A temp home, state root and profile, with fake openclaw and hermes CLIs first on PATH. */ -function rig(target: 'openclaw' | 'hermes', extra: { noCli?: boolean; env?: Record } = {}) { +function rig(target: 'openclaw' | 'hermes', extra: { noCli?: boolean; fresh?: boolean; env?: Record } = {}) { const d = tmp(); const bin = path.join(d, 'bin'); fs.mkdirSync(bin); @@ -59,11 +75,13 @@ function rig(target: 'openclaw' | 'hermes', extra: { noCli?: boolean; env?: Reco fs.writeFileSync(path.join(bin, 'hermes'), FAKE_HERMES, { mode: 0o755 }); const profile = path.join(d, target === 'openclaw' ? 'oc' : 'he'); fs.mkdirSync(profile, { recursive: true }); + const config = path.join(profile, CONFIG[target]); + if (!extra.fresh) fs.writeFileSync(config, HEALTHY[target]); const env = { PATH: (extra.noCli ? '' : bin + path.delimiter) + '/usr/bin:/bin', ANTHROPIC_API_KEY: 'sekret', [target === 'openclaw' ? 'OPENCLAW_STATE_DIR' : 'HERMES_HOME']: profile, ...extra.env }; const o = (more: Partial = {}): RegisterOptions => ({ target: TARGETS[target], checkout: ROOT, stateRoot: path.join(d, 'state'), env, home: path.join(d, 'home'), version: VERSION, ...more }); const root = path.join(profile, target === 'openclaw' ? 'extensions' : 'plugins', HOOK_ID); const calls = () => fs.existsSync(path.join(profile, 'fake-calls')) ? fs.readFileSync(path.join(profile, 'fake-calls'), 'utf8').trim().split('\n') : []; - return { d, bin, profile, env, o, root, calls }; + return { d, bin, profile, config, env, o, root, calls }; } describe('start hook: the note and the plugin files', () => { @@ -486,7 +504,140 @@ describe('start hook: teardown keeps what gstack did not write (fake CLI)', () = }, 300_000); }); +describe('start hook: HARN-18 smoke defects (GST-778)', () => { + const offPath = (r: { env: Record }) => ({ ...r.env, PATH: '/usr/bin:/bin' }); + const cli = (r: { bin: string; env: Record }, t: 'openclaw' | 'hermes', ...args: string[]) => spawnSync(path.join(r.bin, t), args, { env: r.env, encoding: 'utf8', timeout: 30_000 }); + + test('OPENCLAW_CONFIG_PATH naming the profile own openclaw.json, even through a symlink, keeps the hook', () => { + const r = rig('openclaw'); + const link = path.join(tmp(), 'ln'); + fs.symlinkSync(r.profile, link); + const viaEnv = register({ ...r.o(), env: { ...r.env, GSTACK_HARNESS_START_HOOK: '1', OPENCLAW_CONFIG_PATH: path.join(link, 'openclaw.json') } }); + expect(viaEnv.hook).toMatchObject({ state: 'active', required: true }); + const s = rig('openclaw'); + expect(register({ ...s.o({ startHook: true }), env: { ...s.env, OPENCLAW_CONFIG_PATH: s.config } }).hook?.state).toBe('active'); + }, 120_000); + + test('a foreign OPENCLAW_CONFIG_PATH still turns an env opt-in off, and says so as a failure with the fix', () => { + const r = rig('openclaw'); + const res = register({ ...r.o(), env: { ...r.env, GSTACK_HARNESS_START_HOOK: '1', OPENCLAW_CONFIG_PATH: path.join(r.d, 'elsewhere', 'openclaw.json') } }); + expect(res.hook).toMatchObject({ state: 'off', required: true }); + expect(res.hook?.detail).toContain('OPENCLAW_CONFIG_PATH points OpenClaw at ' + path.join(r.d, 'elsewhere', 'openclaw.json')); + expect(res.hook?.fix).toContain('unset OPENCLAW_CONFIG_PATH or set it to ' + r.config); + expect(r.calls()).toEqual([]); + expect(fs.existsSync(path.join(r.profile, 'extensions'))).toBe(false); + expect(readLedger(res.ledger)!.meta.start_hook).toBe('off'); + expect(fs.readFileSync(r.config, 'utf8')).toBe(HEALTHY.openclaw); + const custom = register({ ...r.o({ skillsDir: path.join(r.d, 'custom', 'skills') }), env: { ...r.env, GSTACK_HARNESS_START_HOOK: '1' } }); + expect(custom.hook).toMatchObject({ state: 'off', required: true }); + expect(custom.hook?.fix).toContain('--no-start-hook'); + expect(register({ ...r.o({ skillsDir: path.join(r.d, 'custom', 'skills'), startHook: false }), env: { ...r.env, GSTACK_HARNESS_START_HOOK: '1' } }).hook).toEqual({ state: 'off' }); + }, 120_000); + + test('a fresh harness with no config: turning the hook on creates no config, runs no harness command, and fails with the setup to run first', () => { + for (const t of ['openclaw', 'hermes'] as const) { + const r = rig(t, { fresh: true }); + const res = register(r.o({ startHook: true })); + expect(res.hook).toMatchObject({ state: 'written', required: true }); + expect(res.hook?.detail).toContain('no ' + (t === 'openclaw' ? 'OpenClaw' : 'Hermes') + ' config at ' + r.config); + expect(res.hook?.fix).toContain(t === 'openclaw' ? 'openclaw onboard' : 'hermes setup'); + expect(fs.existsSync(r.config)).toBe(false); + expect(r.calls()).toEqual([]); + expect(fs.existsSync(path.join(r.profile, 'skills'))).toBe(true); + expect(readLedger(res.ledger)!.meta.hook_active).toBe('no'); + const bare = rig(t, { fresh: true, noCli: true }); + const noCli = register(bare.o({ startHook: true })).hook; + expect(noCli).toMatchObject({ state: 'written', required: true }); + expect(noCli?.detail).toContain(t + ' is not on PATH'); + expect(noCli?.fix).toContain(t === 'openclaw' ? 'openclaw onboard' : 'hermes setup'); + expect(noCli?.fix).not.toContain(t === 'openclaw' ? 'config set' : 'plugins enable'); + } + }, 300_000); + + test('once the harness setup has written its config, the next refresh turns the hook on and changes only the plugin entry', () => { + const o = rig('openclaw', { fresh: true }); + register(o.o({ startHook: true })); + fs.writeFileSync(o.config, HEALTHY.openclaw); + expect(register(o.o()).hook).toMatchObject({ state: 'active', retried: true }); + const json = JSON.parse(fs.readFileSync(o.config, 'utf8')); + expect(json.plugins).toEqual({ entries: { [HOOK_ID]: { enabled: true, hooks: { allowConversationAccess: true } } } }); + delete json.plugins; + expect(json).toEqual(JSON.parse(HEALTHY.openclaw)); + expect(o.calls().filter(l => l.startsWith('config set')).map(l => l.split('|')[0])).toEqual(['config set plugins.entries.gstack-start.hooks.allowConversationAccess true', 'config set plugins.entries.gstack-start.enabled true']); + + const h = rig('hermes', { fresh: true }); + register(h.o({ startHook: true })); + fs.writeFileSync(h.config, HEALTHY.hermes); + expect(register(h.o()).hook).toMatchObject({ state: 'active', retried: true }); + expect(fs.readFileSync(h.config, 'utf8')).toBe(HEALTHY.hermes + 'plugins:\n enabled:\n - gstack-start\n'); + }, 300_000); + + test('turning off or unregistering a hook that never turned on, in a harness with no config, creates no config', () => { + for (const t of ['openclaw', 'hermes'] as const) { + const r = rig(t, { fresh: true }); + register(r.o({ startHook: true })); + if (t === 'openclaw') expect(register(r.o({ startHook: false })).hook?.state).toBe('off'); + else unregister(r.o().stateRoot, ROOT, 'hermes', r.env); + expect(fs.existsSync(r.config)).toBe(false); + expect(r.calls()).toEqual([]); + expect(fs.existsSync(r.root)).toBe(false); + } + }, 300_000); + + test('Hermes status without hermes on PATH (its own terminal tool) reads the hook from config.yaml', () => { + const r = rig('hermes'); + expect(register(r.o({ startHook: true })).hook?.state).toBe('active'); + const row = () => status(r.o().stateRoot, VERSION, offPath(r)).find(s => s.target === 'hermes')!.hook!; + expect(row()).toMatchObject({ state: 'active' }); + expect(row().detail).toContain('hermes is not on PATH'); + cli(r, 'hermes', 'plugins', 'disable', HOOK_ID); + expect(row().state).toBe('disabled'); + fs.writeFileSync(r.config, HEALTHY.hermes); + expect(row().state).toBe('written'); + fs.writeFileSync(r.config, 'plugins: [unclosed\n'); + expect(row()).toMatchObject({ state: 'unknown' }); + expect(row().detail).toContain('hermes is not on PATH'); + }, 120_000); + + test('OpenClaw status without openclaw on PATH is unknown when openclaw.json turns the hook on, disabled when the user turned it off', () => { + const r = rig('openclaw'); + expect(register(r.o({ startHook: true })).hook?.state).toBe('active'); + const row = () => status(r.o().stateRoot, VERSION, offPath(r)).find(s => s.target === 'openclaw')!.hook!; + expect(row()).toMatchObject({ state: 'unknown' }); + expect(row().detail).toContain('openclaw is not on PATH'); + cli(r, 'openclaw', 'config', 'set', 'plugins.entries.gstack-start.enabled', 'false'); + expect(row().state).toBe('disabled'); + expect(status(r.o().stateRoot, VERSION, r.env).find(s => s.target === 'openclaw')!.hook!.state).toBe('disabled'); + }, 120_000); +}); + + describe.skipIf(process.platform === 'win32')('start hook through ./setup --host', () => { + test('OPENCLAW_CONFIG_PATH naming the profile own config keeps an env opt-in on; a foreign one exits 4 with the hook off', () => { + const f = makeFixture(); + const src = makeSource(f, path.join(f.dir, 'gstack')); + const bin = path.join(f.dir, 'fakebin'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'openclaw'), FAKE_OPENCLAW, { mode: 0o755 }); + const profile = path.join(f.home, '.openclaw'); + fs.mkdirSync(profile, { recursive: true }); + fs.writeFileSync(path.join(profile, 'openclaw.json'), HEALTHY.openclaw); + const PATH = bin + path.delimiter + f.env.PATH; + const own = runSetup(f, path.join(src, 'setup'), ['--host', 'openclaw'], { env: { PATH, GSTACK_HARNESS_START_HOOK: '1', OPENCLAW_CONFIG_PATH: path.join(profile, 'openclaw.json') } }); + expect(own.status, own.stdout + own.stderr).toBe(0); + expect(own.stdout).toContain('start hook: active'); + const g = makeFixture(); + const gsrc = makeSource(g, path.join(g.dir, 'gstack')); + fs.mkdirSync(path.join(g.home, '.openclaw'), { recursive: true }); + fs.writeFileSync(path.join(g.home, '.openclaw', 'openclaw.json'), HEALTHY.openclaw); + const foreign = runSetup(g, path.join(gsrc, 'setup'), ['--host', 'openclaw'], { env: { PATH: bin + path.delimiter + g.env.PATH, GSTACK_HARNESS_START_HOOK: '1', OPENCLAW_CONFIG_PATH: path.join(g.dir, 'other', 'openclaw.json') } }); + expect(foreign.status, foreign.stdout + foreign.stderr).toBe(4); + expect(foreign.stdout).toContain('start hook: off'); + expect(foreign.stderr).toContain('(HARNESS_HOOK_INACTIVE; docs/troubleshooting.md#harness-hook-inactive)'); + expect(foreign.stderr).toContain('OPENCLAW_CONFIG_PATH'); + expect(fs.existsSync(path.join(g.home, '.openclaw', 'skills', 'gstack', 'SKILL.md'))).toBe(true); + }, 300_000); + test('exits 4 with the skills registered when the hook does not turn on, and the next run retries it', () => { const f = makeFixture(); const src = makeSource(f, path.join(f.dir, 'gstack')); @@ -495,6 +646,7 @@ describe.skipIf(process.platform === 'win32')('start hook through ./setup --host fs.writeFileSync(path.join(bin, 'openclaw'), FAKE_OPENCLAW, { mode: 0o755 }); const profile = path.join(f.home, '.openclaw'); fs.mkdirSync(profile, { recursive: true }); + fs.writeFileSync(path.join(profile, 'openclaw.json'), HEALTHY.openclaw); fs.writeFileSync(path.join(profile, 'fake-fail'), ''); const PATH = bin + path.delimiter + f.env.PATH; const first = runSetup(f, path.join(src, 'setup'), ['--host', 'openclaw'], { env: { PATH, GSTACK_HARNESS_START_HOOK: '1' } });