From ca71ca409561020318d9c3f775c290480880d20b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 4 Nov 2026 16:53:58 -0800 Subject: [PATCH] docs(inventory): record /review R56 HELPS with its unit and home-field caveats (GST-1370) (#4005) * docs(inventory): record /review R56 HELPS with its unit and home-field caveats (GST-1370) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(inventory): regenerate docs/inventory.md (GST-1370) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(inventory): add the development-suite limit to the /review caveats (GST-1370) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(inventory): regenerate docs/inventory.md (GST-1370) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- .../capy-gst-1370-review-helps-ledger.md | 1 + docs/inventory.md | 221 +++++++++--------- docs/inventory/evidence.json | 31 ++- lib/inventory.ts | 8 +- test/inventory.test.ts | 12 +- 6 files changed, 152 insertions(+), 123 deletions(-) create mode 100644 changelog.d/capy-gst-1370-review-helps-ledger.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0f9dc987..72e98612a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -812,7 +812,7 @@ Write or change skill prose with `/write-skill` (edit ladder, levels, `gstack-sk 3. The dynamic template discovery picks it up for every host render and for `gstack/llms.txt`. Two things it does not fill in: a row in `SKILL_REQUIREMENTS` in `lib/doctor-components.ts` (the doctor's `--for ` table; `test/doctor-components.test.ts` fails on a skill directory without one, and the same gen run rerenders `bin/gstack-doctor-components.sh`), and the catalog rows readers find the skill through (README.md, AGENTS.md, `docs/skills.md`, `docs/PROJECT_STRUCTURE.md`, the root `SKILL.md.tmpl` router) 4. Budget it: run `bun test/helpers/capture-context-budget.ts` and commit the refreshed `test/fixtures/context-budget.json` — the context-budget ratchet fails any skill without a ceiling; the catalog budget (`test/catalog-budget.test.ts`) fails until its cap is ratcheted by the protocol in its failure message, and the number is quoted in CLAUDE.md, this file and `docs/CONTRIBUTOR_REFERENCE.md` 5. Commit `{skill}/SKILL.md` (and the generated `{skill}/sections/preamble.md` when the template's `preamble-tier` is 2 or higher), external host output is generated at setup time and gitignored -6. Give it a row in the evidence ledger, `docs/inventory/evidence.json` ([ETHOS.md: 9. Earn Its Place](ETHOS.md#9-earn-its-place)): evidence that it helps on frontier models (`sources`, `models`, `prompt: preset`, `scope`, `masked: true`), or `status: untested` with its `reason`, a concrete `plan` (a `test/fixtures/skill-evals//tasks.json` or an /eval-plan preregistration), an `owner` and a `measure_by` date. `test/inventory.test.ts` fails on a skill without a valid row; `bin/gstack-inventory --check` names the problem, and `bin/gstack-inventory` refreshes the dated snapshot in `docs/inventory.md` +6. Give it a row in the evidence ledger, `docs/inventory/evidence.json` ([ETHOS.md: 9. Earn Its Place](ETHOS.md#9-earn-its-place)): evidence that it helps on frontier models (`sources`, `models`, `prompt: preset`, `scope`, `masked: true`, and `caveats` for any limits on the result), or `status: untested` with its `reason`, a concrete `plan` (a `test/fixtures/skill-evals//tasks.json` or an /eval-plan preregistration), an `owner` and a `measure_by` date. `test/inventory.test.ts` fails on a skill without a valid row; `bin/gstack-inventory --check` names the problem, and `bin/gstack-inventory` refreshes the dated snapshot in `docs/inventory.md` ## Conductor workspaces diff --git a/changelog.d/capy-gst-1370-review-helps-ledger.md b/changelog.d/capy-gst-1370-review-helps-ledger.md new file mode 100644 index 000000000..901e224cc --- /dev/null +++ b/changelog.d/capy-gst-1370-review-helps-ledger.md @@ -0,0 +1 @@ +- **The evidence ledger records /review as helps, with its three caveats beside the status.** R2 at the after pin (gstack-evals #207, superseded by #211) measured /review with the contract Stop hook on Opus 5.5 at gstack 699a41cb: blocking recall 26.2% against 3.6% for plain Opus 5.5 with no gstack, +22.6 points (95% CI +11.0 to +33.7). The `review` row in `docs/inventory/evidence.json` now says helps, with its sources, scope and R48 KEEP. A new `caveats` field, allowed only on helps, no-effect and hurts rows, holds the result's limits, and `docs/inventory.md` and `gstack-skill-bill` print them beside the status: /review and the hook count as one unit (42 of 66 sessions ran the readers only after the hook blocked), 41 of the suite's 42 bugs came from gstack-family reviews, and stacks-r1 is a development suite, so the result makes no release claim. The earlier review results stay under `prior`. diff --git a/docs/inventory.md b/docs/inventory.md index 5cef90170..1c5e0d19d 100644 --- a/docs/inventory.md +++ b/docs/inventory.md @@ -1,12 +1,12 @@ # gstack inventory: every part, its load per run and its evidence -Generated by `bin/gstack-inventory` from [evidence.json](inventory/evidence.json), the hand-kept ledger, and lib/context-bill.ts. Measured at commit cd67c41d8 (2026-11-02). This is a dated snapshot: rerun `bin/gstack-inventory` to refresh it; `bin/gstack-inventory --check` validates the ledger. The rule: [ETHOS.md: 9. Earn Its Place](../ETHOS.md#9-earn-its-place). +Generated by `bin/gstack-inventory` from [evidence.json](inventory/evidence.json), the hand-kept ledger, and lib/context-bill.ts. Measured at commit 56c1fb045 (2026-11-04). This is a dated snapshot: rerun `bin/gstack-inventory` to refresh it; `bin/gstack-inventory --check` validates the ledger. The rule: [ETHOS.md: 9. Earn Its Place](../ETHOS.md#9-earn-its-place). Token counts are offline estimates (estimate: calibrated bytes/token per content class, about ±40%), not billed tokens. | Status | Parts | |---|---| -| helps | 0 | +| helps | 1 | | no-effect | 1 | | hurts | 0 | | untested, with a plan | 17 | @@ -19,48 +19,48 @@ Load per run: a skill's SKILL.md plus the files it must read on every use (`perI | Rank | Part | Kind | Load per run (tokens) | On demand (tokens) | Always-on (tokens) | Loads | Status | Source, plan or reason | |---|---|---|---|---|---|---|---|---| -| 1 | ship-review-panel | shared | 22,830 | n/a | n/a | every /ship run, by depth | untested | plan: GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 1 | ship-review-panel | shared | 23,370 | n/a | n/a | every /ship run, by depth | untested | plan: GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 2 | design-review | skill | 20,159 | 7,778 | 101 | every use | untested | plan: GEVA-2 registry row R56-9 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 3 | ship | skill | 14,587 | 69,678 | 91 | every use | untested | plan: GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 3 | ship | skill | 14,587 | 70,907 | 91 | every use | untested | plan: GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 4 | plan-ceo-review | skill | 14,394 | 21,203 | 313 | every use | untested | plan: GEVA-2 registry row R56-5 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 5 | review | skill | 13,990 | 41,146 | 75 | every use | untested | plan: R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms (grandfathered) | -| 6 | devex-review | skill | 13,911 | 0 | 74 | every use | untested | plan: GEVA-2 registry row R56-11 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 5 | review | skill | 14,268 | 41,459 | 185 | every use | helps | sources: gstack-evals #207 (baa599a9) wiki/board/2026-11-04-geva-271-r2-after-pin-results, gstack-evals #211 (81daa79) wiki/board/2026-11-04-geva-281-r2-after-pin-results-home-field; R48 KEEP; caveats: one unit: /review and the contract Stop hook were measured together, and neither gets credit alone; 42 of 66 gstack sessions ran the review readers only after the hook blocked them (24 unprompted), and that split is not randomized; home-field bias in gstack's favor: 41 of the 42 blocking bugs in stacks-r1 came from gstack-family reviews, so +22.6 may run high; a suite seeded from other reviewers' bugs would remove the bias (GEVA-2 lists it as future work); development suite: stacks-r1 is a development suite, so the result makes no release claim and holds only for these 33 units at these settings | +| 6 | devex-review | skill | 14,025 | 0 | 74 | every use | untested | plan: GEVA-2 registry row R56-11 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 7 | plan-design-review | skill | 13,351 | 12,986 | 79 | every use | untested | plan: GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 8 | retro | skill | 12,257 | 1,514 | 225 | every use | untested | plan: GEVA-2 registry row R56-12 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 8 | retro | skill | 12,262 | 1,515 | 234 | every use | untested | plan: GEVA-2 registry row R56-12 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 9 | plan-devex-review | skill | 11,104 | 20,474 | 80 | every use | untested | plan: GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 10 | autoplan-phases | shared | 10,709 | n/a | n/a | every /autoplan run, by scope | untested | plan: folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered (grandfathered) | | 11 | office-hours | skill | 10,424 | 23,078 | 280 | every use | untested | plan: none (grandfathered) | -| 12 | autoplan | skill | 10,288 | 18,701 | 394 | every use | untested | plan: GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 12 | autoplan | skill | 10,288 | 20,158 | 394 | every use | untested | plan: GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 13 | plan-eng-review | skill | 9,837 | 25,963 | 74 | every use | untested | plan: GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 14 | design-consultation | skill | 9,557 | 10,864 | 282 | every use | untested | plan: GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 15 | setup-gbrain | skill | 9,099 | 10,921 | 106 | every use | untested | plan: none (grandfathered) | +| 15 | setup-gbrain | skill | 9,146 | 10,921 | 106 | every use | untested | plan: none (grandfathered) | | 16 | codex | skill | 8,991 | 10,358 | 68 | every use | untested | plan: none (grandfathered) | -| 17 | sync-gbrain | skill | 8,863 | 0 | 85 | every use | untested | plan: none (grandfathered) | +| 17 | sync-gbrain | skill | 8,886 | 0 | 85 | every use | untested | plan: none (grandfathered) | | 18 | design-shotgun | skill | 8,470 | 1,208 | 248 | every use | untested | plan: none (grandfathered) | -| 19 | plan-tune | skill | 8,377 | 0 | 107 | every use | untested | plan: none (grandfathered) | +| 19 | plan-tune | skill | 8,447 | 0 | 107 | every use | untested | plan: none (grandfathered) | | 20 | spec | skill | 8,305 | 7,759 | 92 | every use | untested | plan: none (grandfathered) | -| 21 | browse | skill | 8,125 | 5,597 | 81 | every use | untested | plan: none (grandfathered) | +| 21 | browse | skill | 8,238 | 5,938 | 81 | every use | untested | plan: none (grandfathered) | | 22 | design-html | skill | 8,096 | 5,844 | 82 | every use | untested | plan: none (grandfathered) | -| 23 | land-and-deploy | skill | 7,652 | 16,188 | 63 | every use | untested | plan: GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 24 | benchmark | skill | 7,430 | 961 | 72 | every use | untested | plan: none (grandfathered) | -| 25 | catalog | shared | 7,291 | n/a | 7,291 | every session | untested | plan: Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description (grandfathered) | -| 26 | investigate | skill | 7,086 | 3,729 | 490 | every use | no-effect | sources: GST-814 (HARN-1, 2026-11-02), org-gstack-gbrain handoffs/HARN-14/hard-follow-2026-11/decision-rule.md; R48 CUT-CANDIDATE | -| 27 | canary | skill | 7,034 | 0 | 66 | every use | untested | plan: none (grandfathered) | -| 28 | qa | skill | 6,951 | 13,877 | 74 | every use | untested | plan: none (grandfathered) | -| 29 | scrape | skill | 6,647 | 0 | 80 | every use | untested | plan: none (grandfathered) | -| 30 | captain | skill | 6,522 | 7,856 | 127 | every use | untested | plan: none (grandfathered) | -| 31 | preamble | shared | 5,743 | n/a | n/a | every tier 2+ skill run, inside that skill's load | untested | plan: I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1 (grandfathered) | -| 32 | gstack-upgrade | skill | 5,726 | 0 | 59 | every use | untested | plan: none (grandfathered) | -| 33 | context-restore | skill | 5,715 | 1,734 | 84 | every use | untested | plan: none (grandfathered) | -| 34 | document-generate | skill | 5,466 | 0 | 110 | every use | untested | plan: none (grandfathered) | -| 35 | ios-qa | skill | 5,368 | 1,378 | 84 | every use | untested | plan: none (grandfathered) | -| 36 | skillify | skill | 5,317 | 0 | 80 | every use | untested | plan: none (grandfathered) | -| 37 | capy-skill | shared | 5,309 | n/a | n/a | every Capy session | untested | plan: none (grandfathered) | -| 38 | health | skill | 5,287 | 0 | 68 | every use | untested | plan: none (grandfathered) | -| 39 | setup-deploy | skill | 5,216 | 0 | 73 | every use | untested | plan: none (grandfathered) | -| 40 | qa-only | skill | 5,205 | 3,460 | 68 | every use | untested | plan: none (grandfathered) | -| 41 | eval-plan | skill | 5,021 | 0 | 108 | every use | untested | plan: none (grandfathered) | -| 42 | gstack | skill | 4,951 | n/a | 59 | every use | untested | plan: none (grandfathered) | +| 23 | land-and-deploy | skill | 7,651 | 16,188 | 71 | every use | untested | plan: GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 24 | catalog | shared | 7,592 | n/a | 7,592 | every session | untested | plan: Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description (grandfathered) | +| 25 | benchmark | skill | 7,430 | 961 | 72 | every use | untested | plan: none (grandfathered) | +| 26 | canary | skill | 7,034 | 0 | 66 | every use | untested | plan: none (grandfathered) | +| 27 | qa | skill | 6,957 | 13,909 | 74 | every use | untested | plan: none (grandfathered) | +| 28 | scrape | skill | 6,647 | 0 | 80 | every use | untested | plan: none (grandfathered) | +| 29 | captain | skill | 6,550 | 9,194 | 127 | every use | untested | plan: none (grandfathered) | +| 30 | preamble | shared | 5,743 | n/a | n/a | every tier 2+ skill run, inside that skill's load | untested | plan: I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1 (grandfathered) | +| 31 | gstack-upgrade | skill | 5,726 | 0 | 59 | every use | untested | plan: none (grandfathered) | +| 32 | context-restore | skill | 5,715 | 1,734 | 84 | every use | untested | plan: none (grandfathered) | +| 33 | document-generate | skill | 5,466 | 0 | 110 | every use | untested | plan: none (grandfathered) | +| 34 | ios-qa | skill | 5,368 | 1,378 | 84 | every use | untested | plan: none (grandfathered) | +| 35 | skillify | skill | 5,317 | 0 | 80 | every use | untested | plan: none (grandfathered) | +| 36 | capy-skill | shared | 5,315 | n/a | n/a | every Capy session | untested | plan: none (grandfathered) | +| 37 | setup-deploy | skill | 5,216 | 0 | 73 | every use | untested | plan: none (grandfathered) | +| 38 | qa-only | skill | 5,205 | 3,460 | 68 | every use | untested | plan: none (grandfathered) | +| 39 | eval-plan | skill | 5,107 | 0 | 108 | every use | untested | plan: none (grandfathered) | +| 40 | investigate | skill | 5,059 | 3,729 | 598 | every use | no-effect | sources: GST-814 (HARN-1, 2026-11-02), org-gstack-gbrain handoffs/HARN-14/hard-follow-2026-11/decision-rule.md; R48 CUT-CANDIDATE | +| 41 | health | skill | 5,041 | 0 | 68 | every use | untested | plan: none (grandfathered) | +| 42 | gstack | skill | 4,990 | n/a | 59 | every use | untested | plan: none (grandfathered) | | 43 | pair-agent | skill | 4,811 | 0 | 63 | every use | untested | plan: none (grandfathered) | | 44 | make-pdf | skill | 4,730 | 1,767 | 65 | every use | untested | plan: none (grandfathered) | | 45 | context-save | skill | 4,637 | 0 | 65 | every use | untested | plan: none (grandfathered) | @@ -71,25 +71,26 @@ Load per run: a skill's SKILL.md plus the files it must read on every use (`perI | 50 | document-release | skill | 4,049 | 10,260 | 72 | every use | untested | plan: none (grandfathered) | | 51 | diagram | skill | 3,895 | 0 | 112 | every use | untested | plan: none (grandfathered) | | 52 | benchmark-models | skill | 3,822 | 0 | 77 | every use | untested | plan: none (grandfathered) | -| 53 | ubicloud | skill | 3,593 | 0 | 83 | every use | untested | plan: none (grandfathered) | -| 54 | modernize | skill | 3,429 | 0 | 106 | every use | untested | plan: none (grandfathered) | -| 55 | setup-browser-cookies | skill | 3,184 | 0 | 79 | every use | untested | plan: none (grandfathered) | -| 56 | deslop-shared-libs | skill | 3,100 | 0 | 64 | every use | untested | plan: none (grandfathered) | -| 57 | landing-report | skill | 2,971 | 0 | 71 | every use | untested | plan: none (grandfathered) | -| 58 | ship-docs-audit | shared | 2,730 | n/a | n/a | every /ship run | untested | plan: folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered (grandfathered) | -| 59 | verify | skill | 2,679 | 0 | 91 | every use | untested | plan: none (grandfathered) | -| 60 | ios-design-review | skill | 2,367 | 0 | 76 | every use | untested | plan: none (grandfathered) | -| 61 | ios-sync | skill | 2,240 | 0 | 90 | every use | untested | plan: none (grandfathered) | -| 62 | ios-clean | skill | 2,214 | 0 | 86 | every use | untested | plan: none (grandfathered) | -| 63 | write-skill | skill | 2,158 | 795 | 100 | every use | untested | plan: none (grandfathered) | -| 64 | ios-fix | skill | 2,096 | 0 | 69 | every use | untested | plan: none (grandfathered) | -| 65 | learn | skill | 1,989 | 0 | 66 | every use | untested | plan: none (grandfathered) | -| 66 | careful | skill | 1,126 | 0 | 171 | every use | safety | reason: safety guard: warns before destructive commands (rm -rf, DROP TABLE, force-push); kept on that reason, an average score cannot see the rare bad run it stops | -| 67 | freeze | skill | 1,043 | 0 | 234 | every use | safety | reason: safety guard: hard-blocks edits outside one directory; kept on that reason | -| 68 | guard | skill | 908 | 0 | 348 | every use | safety | reason: safety guard: careful and freeze together; kept on that reason | -| 69 | capy-factory | shared | 846 | n/a | n/a | every Capy session | untested | plan: none (grandfathered) | -| 70 | unfreeze | skill | 372 | 0 | 65 | every use | safety | reason: safety guard: the off switch for freeze's edit restriction; kept with the guard it releases | -| 71 | claude-code | skill | n/a | n/a | n/a | every use | untested | plan: none (grandfathered) | +| 53 | runaway-stopper | skill | 3,672 | 571 | 78 | every use | untested | plan: docs/evals/runaway-stopper-prereg.md (owner GStack Minder (GSTA-36), measure by 2026-11-24) | +| 54 | ubicloud | skill | 3,638 | 0 | 83 | every use | untested | plan: none (grandfathered) | +| 55 | modernize | skill | 3,418 | 0 | 94 | every use | untested | plan: none (grandfathered) | +| 56 | setup-browser-cookies | skill | 3,289 | 0 | 79 | every use | untested | plan: none (grandfathered) | +| 57 | deslop-shared-libs | skill | 3,100 | 0 | 64 | every use | untested | plan: none (grandfathered) | +| 58 | landing-report | skill | 2,971 | 0 | 71 | every use | untested | plan: none (grandfathered) | +| 59 | ship-docs-audit | shared | 2,730 | n/a | n/a | every /ship run | untested | plan: folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered (grandfathered) | +| 60 | verify | skill | 2,678 | 0 | 91 | every use | untested | plan: none (grandfathered) | +| 61 | ios-design-review | skill | 2,367 | 0 | 76 | every use | untested | plan: none (grandfathered) | +| 62 | ios-sync | skill | 2,240 | 0 | 90 | every use | untested | plan: none (grandfathered) | +| 63 | ios-clean | skill | 2,214 | 0 | 86 | every use | untested | plan: none (grandfathered) | +| 64 | write-skill | skill | 2,158 | 795 | 100 | every use | untested | plan: none (grandfathered) | +| 65 | ios-fix | skill | 2,096 | 0 | 69 | every use | untested | plan: none (grandfathered) | +| 66 | learn | skill | 1,989 | 0 | 66 | every use | untested | plan: none (grandfathered) | +| 67 | careful | skill | 1,126 | 0 | 171 | every use | safety | reason: safety guard: warns before destructive commands (rm -rf, DROP TABLE, force-push); kept on that reason, an average score cannot see the rare bad run it stops | +| 68 | freeze | skill | 1,043 | 0 | 234 | every use | safety | reason: safety guard: hard-blocks edits outside one directory; kept on that reason | +| 69 | guard | skill | 908 | 0 | 348 | every use | safety | reason: safety guard: careful and freeze together; kept on that reason | +| 70 | capy-factory | shared | 884 | n/a | n/a | every Capy session | untested | plan: none (grandfathered) | +| 71 | unfreeze | skill | 372 | 0 | 65 | every use | safety | reason: safety guard: the off switch for freeze's edit restriction; kept with the guard it releases | +| 72 | claude-code | skill | n/a | n/a | n/a | every use | untested | plan: none (grandfathered) | ## Measure next @@ -97,77 +98,77 @@ Untested parts, largest load per run first. A fair test runs the claude_code pre | Order | Part | Load per run (tokens) | Plan | |---|---|---|---| -| 1 | ship-review-panel | 22,830 | GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 1 | ship-review-panel | 23,370 | GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 2 | design-review | 20,159 | GEVA-2 registry row R56-9 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 3 | ship | 14,587 | GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | | 4 | plan-ceo-review | 14,394 | GEVA-2 registry row R56-5 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 5 | review | 13,990 | R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms (grandfathered) | -| 6 | devex-review | 13,911 | GEVA-2 registry row R56-11 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 7 | plan-design-review | 13,351 | GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 8 | retro | 12,257 | GEVA-2 registry row R56-12 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 9 | plan-devex-review | 11,104 | GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 10 | autoplan-phases | 10,709 | folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered (grandfathered) | -| 11 | office-hours | 10,424 | none (grandfathered) | -| 12 | autoplan | 10,288 | GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 13 | plan-eng-review | 9,837 | GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 14 | design-consultation | 9,557 | GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | -| 15 | setup-gbrain | 9,099 | none (grandfathered) | -| 16 | codex | 8,991 | none (grandfathered) | -| 17 | sync-gbrain | 8,863 | none (grandfathered) | -| 18 | design-shotgun | 8,470 | none (grandfathered) | -| 19 | plan-tune | 8,377 | none (grandfathered) | -| 20 | spec | 8,305 | none (grandfathered) | -| 21 | browse | 8,125 | none (grandfathered) | -| 22 | design-html | 8,096 | none (grandfathered) | -| 23 | land-and-deploy | 7,652 | GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 5 | devex-review | 14,025 | GEVA-2 registry row R56-11 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 6 | plan-design-review | 13,351 | GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 7 | retro | 12,262 | GEVA-2 registry row R56-12 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 8 | plan-devex-review | 11,104 | GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 9 | autoplan-phases | 10,709 | folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered (grandfathered) | +| 10 | office-hours | 10,424 | none (grandfathered) | +| 11 | autoplan | 10,288 | GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 12 | plan-eng-review | 9,837 | GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 13 | design-consultation | 9,557 | GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 14 | setup-gbrain | 9,146 | none (grandfathered) | +| 15 | codex | 8,991 | none (grandfathered) | +| 16 | sync-gbrain | 8,886 | none (grandfathered) | +| 17 | design-shotgun | 8,470 | none (grandfathered) | +| 18 | plan-tune | 8,447 | none (grandfathered) | +| 19 | spec | 8,305 | none (grandfathered) | +| 20 | browse | 8,238 | none (grandfathered) | +| 21 | design-html | 8,096 | none (grandfathered) | +| 22 | land-and-deploy | 7,651 | GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered) (grandfathered) | +| 23 | catalog | 7,592 | Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description (grandfathered) | | 24 | benchmark | 7,430 | none (grandfathered) | -| 25 | catalog | 7,291 | Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description (grandfathered) | -| 26 | canary | 7,034 | none (grandfathered) | -| 27 | qa | 6,951 | none (grandfathered) | -| 28 | scrape | 6,647 | none (grandfathered) | -| 29 | captain | 6,522 | none (grandfathered) | -| 30 | preamble | 5,743 | I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1 (grandfathered) | -| 31 | gstack-upgrade | 5,726 | none (grandfathered) | -| 32 | context-restore | 5,715 | none (grandfathered) | -| 33 | document-generate | 5,466 | none (grandfathered) | -| 34 | ios-qa | 5,368 | none (grandfathered) | -| 35 | skillify | 5,317 | none (grandfathered) | -| 36 | capy-skill | 5,309 | none (grandfathered) | -| 37 | health | 5,287 | none (grandfathered) | -| 38 | setup-deploy | 5,216 | none (grandfathered) | -| 39 | qa-only | 5,205 | none (grandfathered) | -| 40 | eval-plan | 5,021 | none (grandfathered) | -| 41 | gstack | 4,951 | none (grandfathered) | -| 42 | pair-agent | 4,811 | none (grandfathered) | -| 43 | make-pdf | 4,730 | none (grandfathered) | -| 44 | context-save | 4,637 | none (grandfathered) | -| 45 | open-gstack-browser | 4,429 | none (grandfathered) | -| 46 | design-motion | 4,295 | none (grandfathered) | -| 47 | test-audit | 4,201 | none (grandfathered) | -| 48 | document-release | 4,049 | none (grandfathered) | -| 49 | diagram | 3,895 | none (grandfathered) | -| 50 | benchmark-models | 3,822 | none (grandfathered) | -| 51 | ubicloud | 3,593 | none (grandfathered) | -| 52 | modernize | 3,429 | none (grandfathered) | -| 53 | setup-browser-cookies | 3,184 | none (grandfathered) | +| 25 | canary | 7,034 | none (grandfathered) | +| 26 | qa | 6,957 | none (grandfathered) | +| 27 | scrape | 6,647 | none (grandfathered) | +| 28 | captain | 6,550 | none (grandfathered) | +| 29 | preamble | 5,743 | I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1 (grandfathered) | +| 30 | gstack-upgrade | 5,726 | none (grandfathered) | +| 31 | context-restore | 5,715 | none (grandfathered) | +| 32 | document-generate | 5,466 | none (grandfathered) | +| 33 | ios-qa | 5,368 | none (grandfathered) | +| 34 | skillify | 5,317 | none (grandfathered) | +| 35 | capy-skill | 5,315 | none (grandfathered) | +| 36 | setup-deploy | 5,216 | none (grandfathered) | +| 37 | qa-only | 5,205 | none (grandfathered) | +| 38 | eval-plan | 5,107 | none (grandfathered) | +| 39 | health | 5,041 | none (grandfathered) | +| 40 | gstack | 4,990 | none (grandfathered) | +| 41 | pair-agent | 4,811 | none (grandfathered) | +| 42 | make-pdf | 4,730 | none (grandfathered) | +| 43 | context-save | 4,637 | none (grandfathered) | +| 44 | open-gstack-browser | 4,429 | none (grandfathered) | +| 45 | design-motion | 4,295 | none (grandfathered) | +| 46 | test-audit | 4,201 | none (grandfathered) | +| 47 | document-release | 4,049 | none (grandfathered) | +| 48 | diagram | 3,895 | none (grandfathered) | +| 49 | benchmark-models | 3,822 | none (grandfathered) | +| 50 | runaway-stopper | 3,672 | docs/evals/runaway-stopper-prereg.md (owner GStack Minder (GSTA-36), measure by 2026-11-24) | +| 51 | ubicloud | 3,638 | none (grandfathered) | +| 52 | modernize | 3,418 | none (grandfathered) | +| 53 | setup-browser-cookies | 3,289 | none (grandfathered) | | 54 | deslop-shared-libs | 3,100 | none (grandfathered) | | 55 | landing-report | 2,971 | none (grandfathered) | | 56 | ship-docs-audit | 2,730 | folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered (grandfathered) | -| 57 | verify | 2,679 | none (grandfathered) | +| 57 | verify | 2,678 | none (grandfathered) | | 58 | ios-design-review | 2,367 | none (grandfathered) | | 59 | ios-sync | 2,240 | none (grandfathered) | | 60 | ios-clean | 2,214 | none (grandfathered) | | 61 | write-skill | 2,158 | none (grandfathered) | | 62 | ios-fix | 2,096 | none (grandfathered) | | 63 | learn | 1,989 | none (grandfathered) | -| 64 | capy-factory | 846 | none (grandfathered) | +| 64 | capy-factory | 884 | none (grandfathered) | | 65 | claude-code | n/a | none (grandfathered) | ## Shared parts and preamble tiers | Tier | Skills | What loads | |---|---|---| -| 1 | benchmark, benchmark-models, browse, diagram, gstack, make-pdf, open-gstack-browser, scrape, setup-browser-cookies, verify | a short preamble inline in SKILL.md | +| 1 | benchmark, benchmark-models, browse, diagram, gstack, make-pdf, open-gstack-browser, runaway-stopper, scrape, setup-browser-cookies, verify | a short preamble inline in SKILL.md | | 2 | canary, captain, context-restore, context-save, design-html, design-motion, design-shotgun, document-generate, document-release, eval-plan, health, investigate, ios-clean, ios-design-review, ios-fix, ios-sync, landing-report, learn, modernize, pair-agent, plan-tune, retro, setup-deploy, setup-gbrain, skillify, sync-gbrain, test-audit, ubicloud, write-skill | the shared sections/preamble.md (row preamble), once per session | | 3 | autoplan, claude-code, codex, design-consultation, devex-review, ios-qa, office-hours, plan-ceo-review, plan-design-review, plan-devex-review, plan-eng-review, spec | the shared sections/preamble.md (row preamble), once per session | | 4 | design-review, land-and-deploy, qa, qa-only, review, ship | the shared sections/preamble.md (row preamble), once per session | @@ -187,11 +188,11 @@ Each count is quoted from the file and line named; `bin/gstack-inventory --check | Skill | What | Count | Source | |---|---|---|---| -| ship | Step 9 review, pass 1, light depth, docs or tests only | 1 reader (the core checklist) | docs/review-policy.md:194 | -| ship | Step 9 review, pass 1, light depth, code | 3 readers (core, one batched specialist, native adversarial), plus the outside voice on a diff that publishes numbers | docs/review-policy.md:195 | -| ship | Step 9 review, pass 1, standard depth | the batched specialist, the Red Team at 200 runtime lines, adversarial and outside voice | docs/review-policy.md:196 | -| ship | Step 9 review, pass 1, full depth | the batched specialist on the whole diff and the Red Team always | docs/review-policy.md:197 | -| ship | targeted reader on money, gate, auth or secret code | +1 at every depth | docs/review-policy.md:319 | +| ship | Step 9 review, pass 1, light depth, docs or tests only | 1 reader (the core checklist) | docs/review-policy.md:295 | +| ship | Step 9 review, pass 1, light depth, code | 3 readers (core, one batched specialist, native adversarial), plus the outside voice on a diff that publishes numbers | docs/review-policy.md:296 | +| ship | Step 9 review, pass 1, standard depth | the batched specialist, the Red Team at 200 runtime lines, adversarial and outside voice | docs/review-policy.md:297 | +| ship | Step 9 review, pass 1, full depth | the batched specialist on the whole diff and the Red Team always | docs/review-policy.md:298 | +| ship | targeted reader on money, gate, auth or secret code | +1 at every depth | docs/review-policy.md:446 | | ship | audits (Steps 7, 8 and 14.5) | 3 | lib/ship-budget-calibration.json:3 | | ship | measured: one /ship child (gstack-evals#31) | 10 review dispatches | lib/ship-budget-calibration.json:43 | | ship | prior cost of one dispatch on the Capy meter | 7,430,000 tokens | lib/ship-budget-calibration.json:56 | @@ -234,9 +235,9 @@ Each count is quoted from the file and line named; `bin/gstack-inventory --check ## Key list and aliases -71 keys: 64 skills (the folders `discoverTemplates()` finds, the root skill as `gstack`) and 7 shared parts. +72 keys: 65 skills (the folders `discoverTemplates()` finds, the root skill as `gstack`) and 7 shared parts. -- Skills: autoplan, benchmark, benchmark-models, browse, canary, captain, careful, claude-code, codex, context-restore, context-save, cso, design-consultation, design-html, design-motion, design-review, design-shotgun, deslop-shared-libs, devex-review, diagram, document-generate, document-release, eval-plan, freeze, gstack, gstack-upgrade, guard, health, investigate, ios-clean, ios-design-review, ios-fix, ios-qa, ios-sync, land-and-deploy, landing-report, learn, make-pdf, modernize, office-hours, open-gstack-browser, pair-agent, plan-ceo-review, plan-design-review, plan-devex-review, plan-eng-review, plan-tune, qa, qa-only, retro, review, scrape, setup-browser-cookies, setup-deploy, setup-gbrain, ship, skillify, spec, sync-gbrain, test-audit, ubicloud, unfreeze, verify, write-skill. +- Skills: autoplan, benchmark, benchmark-models, browse, canary, captain, careful, claude-code, codex, context-restore, context-save, cso, design-consultation, design-html, design-motion, design-review, design-shotgun, deslop-shared-libs, devex-review, diagram, document-generate, document-release, eval-plan, freeze, gstack, gstack-upgrade, guard, health, investigate, ios-clean, ios-design-review, ios-fix, ios-qa, ios-sync, land-and-deploy, landing-report, learn, make-pdf, modernize, office-hours, open-gstack-browser, pair-agent, plan-ceo-review, plan-design-review, plan-devex-review, plan-eng-review, plan-tune, qa, qa-only, retro, review, runaway-stopper, scrape, setup-browser-cookies, setup-deploy, setup-gbrain, ship, skillify, spec, sync-gbrain, test-audit, ubicloud, unfreeze, verify, write-skill. - Shared: preamble, catalog, capy-skill, capy-factory, ship-review-panel, ship-docs-audit, autoplan-phases. - Installed names: `gstack-` is key `` (`gstack-ship` is `ship`); `gstack-upgrade` keeps its name; the root skill installs as `gstack`. - Other names (the ledger's `aliases`): gstack (root skill) → gstack; upgrade → gstack-upgrade; preamble (ceremony) → preamble; always-on skill catalog (descriptions) → catalog; capy/skill/gstack/SKILL.md (Capy context) → capy-skill; capy/skill/gstack/factory.md (Capy context) → capy-factory; ship: review panel (specialist, adversarial, targeted, outside voice, red team) → ship-review-panel; ship: docs audit (Step 14.5) → ship-docs-audit; autoplan: CEO, design, DX and eng phases, outside voice → autoplan-phases. diff --git a/docs/inventory/evidence.json b/docs/inventory/evidence.json index d85ce971a..e724304f0 100644 --- a/docs/inventory/evidence.json +++ b/docs/inventory/evidence.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.", + "note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; caveats lists a verdict's limits, shown beside its status; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.", "seed_source": "org-gstack-gbrain/research/gstack-part-evidence.json (GEVA-156 for GST-915, 2026-11-02, gstack-evals main 409ff47)", "parts": { "autoplan": { @@ -420,9 +420,27 @@ "seed": "GEVA-2, 2026-11-02: retro: untested (never measured)" }, "review": { - "status": "untested", + "status": "helps", "grandfathered": true, - "plan": "R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms", + "sources": [ + "gstack-evals #207 (baa599a9) wiki/board/2026-11-04-geva-271-r2-after-pin-results", + "gstack-evals #211 (81daa79) wiki/board/2026-11-04-geva-281-r2-after-pin-results-home-field" + ], + "models": [ + "claude-opus-5-5" + ], + "prompt": "preset", + "scope": "/review as a Claude Code user installs it at gstack 699a41cb (./setup --host claude), contract Stop hook included, told /review, against plain Opus 5.5 with no gstack; Claude Code CLI 2.1.314 on the claude_code preset; stacks-r1, 33 units, 42 blocking bugs, 2 passes: blocking recall 22/84 (26.2%) vs 3/84 (3.6%), +22.6 points [95% PR-clustered CI +11.0, +33.7], MDD 24.5", + "masked": true, + "r48": { + "verdict": "KEEP", + "ref": "gstack-evals #211, wiki/board/2026-11-04-geva-281-r2-after-pin-results-home-field (What we found: R48 KEEP, rr-decide.py KEEP)" + }, + "caveats": [ + "one unit: /review and the contract Stop hook were measured together, and neither gets credit alone; 42 of 66 gstack sessions ran the review readers only after the hook blocked them (24 unprompted), and that split is not randomized", + "home-field bias in gstack's favor: 41 of the 42 blocking bugs in stacks-r1 came from gstack-family reviews, so +22.6 may run high; a suite seeded from other reviewers' bugs would remove the bias (GEVA-2 lists it as future work)", + "development suite: stacks-r1 is a development suite, so the result makes no release claim and holds only for these 33 units at these settings" + ], "prior": [ { "ref": "gstack-evals #108 stacks round 1 review (GEVA-89), wiki/board/2026-11-01-geva-89-stacks-round-1-review-results", @@ -450,11 +468,8 @@ "note": "/review's prompts through the Messages API; Opus recall 44% vs 39% plain, 12 bugs" } ], - "r48": { - "verdict": "FIX-METHOD", - "ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R2 is the fix" - }, - "seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)" + "seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)", + "note": "R2 at the after pin: GEVA-271 under prereg #194 (amending #154), runs review-units-20261104T070239Z-cbff (gstack-full) and review-units-20261104T123631Z-3fb3 (none-rr); #211 (GEVA-281) supersedes #207's post with every number unchanged and adds the home-field limit. Until then, this row was untested with R48 FIX-METHOD (org-gstack-gbrain/research/why-null-stacks-r1.md, GEVA-100), plan R2 (gstack-evals #129). R2's first two attempts gave no outcome: GEVA-175 (procedure gate 10 of 19) and GEVA-229 (exposure smoke 2 of 3). masked: the no-gstack arm ran in the rig's Docker sandbox with no gstack install, and #207's read audit of its 66 transcripts found no read of ~/.capy/drive or ~/.local/share/gstack. Only Opus 5.5 was measured; GPT-6.1 Sol was not." }, "scrape": { "status": "untested", diff --git a/lib/inventory.ts b/lib/inventory.ts index 9615e4841..ef05c437f 100644 --- a/lib/inventory.ts +++ b/lib/inventory.ts @@ -66,7 +66,7 @@ export const PRIOR_WHY: Record = { 'not-claude-code': 'ran outside Claude Code (Messages API, Codex or Capy)', 'variant-only': 'variant against variant, no no-gstack arm', }; -const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'seed', 'aliases', 'note']; +const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'caveats', 'seed', 'aliases', 'note']; export type LedgerCode = | 'LEDGER_PARSE' | 'LEDGER_KEY_MISSING' | 'LEDGER_KEY_EXTRA' | 'LEDGER_STATUS_INVALID' | 'LEDGER_FIELD_UNKNOWN' @@ -75,7 +75,7 @@ export type LedgerCode = | 'LEDGER_REASON_MISSING' | 'LEDGER_OWNER_MISSING' | 'LEDGER_MEASURE_BY_MISSING' | 'LEDGER_PRIOR_INVALID' | 'LEDGER_FIELD_INVALID' | 'INVENTORY_SOURCE_MISSING' | 'INVENTORY_PART_MISSING'; export interface LedgerProblem { key: string; code: LedgerCode; message: string } -export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; seed?: string; aliases?: string[]; note?: string } +export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; caveats?: string[]; seed?: string; aliases?: string[]; note?: string } export interface Ledger { parts: Record; unmapped?: Array<{ row: string; why: string }>; [k: string]: unknown } const isObj = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); @@ -100,6 +100,8 @@ export function validateEntry(key: string, entry: unknown): LedgerProblem[] { add('LEDGER_PRIOR_INVALID', `prior needs [{ref, why: [...]}] with why from: ${Object.keys(PRIOR_WHY).join(', ')}`); if (entry.aliases !== undefined && !(Array.isArray(entry.aliases) && entry.aliases.length > 0 && entry.aliases.every(str))) add('LEDGER_FIELD_INVALID', 'aliases needs a list of names'); const status = entry.status; + if (entry.caveats !== undefined && !((status === 'helps' || status === 'no-effect' || status === 'hurts') && Array.isArray(entry.caveats) && entry.caveats.length > 0 && entry.caveats.every(str))) + add('LEDGER_FIELD_INVALID', 'caveats is a list of limits on a helps, no-effect or hurts result'); if (status === 'helps' || status === 'no-effect' || status === 'hurts') { if (!Array.isArray(entry.sources) || !entry.sources.length || !entry.sources.every(str)) add('LEDGER_SOURCES_MISSING', `${status} needs sources: one or more run ids or results files`); if (!frontierModels(entry.models)) add('LEDGER_MODELS_NOT_FRONTIER', `models must include ${FRONTIER_MODELS.join(' or ')} plus at most one named low-cost primary, never Haiku or Sonnet (R54)`); @@ -164,7 +166,7 @@ export const keyOf = (name: string) => (name !== 'gstack-upgrade' && name.starts export function evidenceRef(e: LedgerEntry): string { if (e.status === 'untested') return `plan: ${e.plan}${e.grandfathered ? ' (grandfathered)' : ` (owner ${e.owner}, measure by ${e.measure_by})`}`; if (e.status === 'safety') return `reason: ${e.reason}`; - return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}`; + return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}${e.caveats ? `; caveats: ${e.caveats.join('; ')}` : ''}`; } /** gstack-skill-bill's EVIDENCE line for one skill folder in a gstack checkout. */ diff --git a/test/inventory.test.ts b/test/inventory.test.ts index a26e5e64d..958d75e8d 100644 --- a/test/inventory.test.ts +++ b/test/inventory.test.ts @@ -9,7 +9,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { discoverTemplates } from '../scripts/discover-skills'; -import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, installReason, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory'; +import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, evidenceRef, installReason, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory'; const ROOT = path.resolve(import.meta.dir, '..'); const BIN = path.join(ROOT, 'bin', 'gstack-inventory'); @@ -129,6 +129,16 @@ describe('the per-status schema', () => { expect(codes('a', { status: 'safety', reason: 'x', aliases: [] })).toEqual(['LEDGER_FIELD_INVALID']); expect(codes('a', { status: 'safety', reason: 'x', aliases: ['other-name'] })).toEqual([]); }); + + test('caveats is a list of limits on a verdict row, shown beside its sources', () => { + const caveats = ['measured with the hook', 'suite seeded from its own reviews']; + expect(codes('a', { status: 'helps', ...VERDICT, caveats })).toEqual([]); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: [] })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: 'one string' })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: [''] })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', caveats })).toEqual(['LEDGER_FIELD_INVALID']); + expect(evidenceRef({ status: 'helps', ...VERDICT, caveats } as any)).toBe('sources: gstack-evals run r9 results.json; caveats: measured with the hook; suite seeded from its own reviews'); + }); }); /** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */