From 37f515bae0eaa0e91764a0223f52ac92287425ee Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 2 Nov 2026 05:03:32 +0000 Subject: [PATCH] inventory: the R56 evidence ledger, gstack-inventory and skill-bill EVIDENCE docs/inventory/evidence.json holds one row per inventory key: the skill folders discoverTemplates() finds (root skill as gstack, symlinked folders excluded) plus seven shared parts, 71 keys. lib/inventory.ts validates each row by its status's own fields, with named codes (LEDGER_*), and joins the ledger with lib/context-bill.ts buildBill. bin/gstack-inventory writes docs/inventory.md, --check validates (exit 1 with codes), --json prints. Paid dispatches per run for /ship and /autoplan are quoted from their own docs and found by line, so a moved citation fails --check (INVENTORY_SOURCE_MISSING). Every current row is grandfathered and seeded from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json): in-flight fair tests and proposed registry rows as plans, earlier unfair results under prior with why (stub, unmasked, ceiling, ...), five safety rows. gstack-skill-bill prints the skill's EVIDENCE line. Evidence: plan none (owner ruling R56, GST-914); the ledger itself is the measurement record Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-inventory | 41 ++ bin/gstack-skill-bill | 3 +- docs/inventory/evidence.json | 623 +++++++++++++++++++++++++ lib/inventory.ts | 313 +++++++++++++ lib/skill-bill.ts | 8 +- test/fixtures/module-size-ratchet.json | 1 + test/inventory.test.ts | 188 ++++++++ test/skill-bill.test.ts | 35 +- 8 files changed, 1208 insertions(+), 4 deletions(-) create mode 100755 bin/gstack-inventory create mode 100644 docs/inventory/evidence.json create mode 100644 lib/inventory.ts create mode 100644 test/inventory.test.ts diff --git a/bin/gstack-inventory b/bin/gstack-inventory new file mode 100755 index 000000000..423f046de --- /dev/null +++ b/bin/gstack-inventory @@ -0,0 +1,41 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +// gstack-inventory — every gstack part, its load per run and its evidence (ETHOS.md: 9. Earn Its Place). Logic: lib/inventory.ts. +import { writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { buildInventory, checkInventory, DOC_PATH, LEDGER_PATH, renderInventory } from "../lib/inventory"; + +const USAGE = `Usage: gstack-inventory [--root ] [--check | --json] + gstack-inventory --help + +Joins the evidence ledger (${LEDGER_PATH}) with gstack-context-bill's sizes and writes ${DOC_PATH}: +parts ranked by load per run, each part's evidence status, a "measure next" order (largest untested +first), shared-part sizes, paid dispatches per run for /ship and /autoplan, and the key list. +--check validates the ledger (every key has one row; each row has its status's fields), the quoted +dispatch sources and the shared-part files, and writes nothing. --json prints the inventory. +Exit 0 when clean; 1 with one ": ()" line per problem; 2 on a bad argument.`; + +const args = process.argv.slice(2); +if (args.includes("--help") || args.includes("-h")) { console.log(USAGE); process.exit(0); } +const rootAt = args.indexOf("--root"); +const root = resolve(rootAt >= 0 ? args[rootAt + 1] ?? "" : join(import.meta.dir, "..")); +const unknown = args.find((a, i) => !["--check", "--json", "--root"].includes(a) && !(rootAt >= 0 && i === rootAt + 1)); +if (unknown !== undefined || (rootAt >= 0 && !args[rootAt + 1]) || (args.includes("--check") && args.includes("--json"))) { + console.error(unknown !== undefined ? `gstack-inventory: unknown argument '${unknown}'; valid: --root --check --json --help` : USAGE); + process.exit(2); +} +const report = (problems: { key: string; code: string; message: string }[]) => { + for (const p of problems) console.error(`gstack-inventory: ${p.key}: ${p.message} (${p.code})`); +}; +if (args.includes("--check")) { + const checked = checkInventory(root); + report(checked.problems); + if (checked.problems.length) process.exit(1); + console.log(`INVENTORY: ok keys=${checked.keys.length} ledger=${LEDGER_PATH} dispatch_sources=${checked.dispatches.length}`); + process.exit(0); +} +const inv = buildInventory(root); +report(inv.problems); +if (args.includes("--json")) { console.log(JSON.stringify(inv, null, 2)); process.exit(inv.problems.length ? 1 : 0); } +if (inv.problems.length) { console.error(`gstack-inventory: ${DOC_PATH} not written; fix the problems above, then rerun`); process.exit(1); } +writeFileSync(join(root, DOC_PATH), renderInventory(inv)); +console.log(`INVENTORY: wrote ${DOC_PATH} keys=${inv.keys.length} commit=${inv.commit}`); diff --git a/bin/gstack-skill-bill b/bin/gstack-skill-bill index 86aa1c657..bb6e6d755 100755 --- a/bin/gstack-skill-bill +++ b/bin/gstack-skill-bill @@ -11,7 +11,8 @@ Names the skill's kind: gstack-checkout (edit SKILL.md.tmpl, then bun run gen:sk gstack-install (never edit; gstack-upgrade overwrites it) or user. BASELINE is the newest commit with a "Skill-Diet: " trailer; DUE means the authored file grew over 10% since then. In a gstack checkout, EVAL TASKS names test/fixtures/skill-evals//tasks.json (or none) and -EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format. +EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format, and +EVIDENCE prints the skill's row in docs/inventory/evidence.json: its status and source, plan or reason. A name must match exactly; pass a path when it names more than one skill. Exit 0 on a report, 2 on SKILL_NOT_FOUND, SKILL_AMBIGUOUS or SKILL_MD_MISSING (each with a fix line).`; diff --git a/docs/inventory/evidence.json b/docs/inventory/evidence.json new file mode 100644 index 000000000..06f89f1b0 --- /dev/null +++ b/docs/inventory/evidence.json @@ -0,0 +1,623 @@ +{ + "schema_version": 1, + "note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.", + "seed_source": "org-gstack-gbrain/research/gstack-part-evidence.json (GEVA-156 for GST-915, 2026-11-02, gstack-evals main 409ff47)", + "parts": { + "autoplan": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: autoplan: untested (never measured)" + }, + "benchmark": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: benchmark: untested (never measured)" + }, + "benchmark-models": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: benchmark-models: untested (never measured)" + }, + "browse": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: browse: untested (never measured)" + }, + "canary": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: canary: untested (never measured)" + }, + "captain": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: captain: untested (never measured)" + }, + "careful": { + "status": "safety", + "grandfathered": true, + "reason": "safety guard: warns before destructive commands (rm -rf, DROP TABLE, force-push); kept on that reason, an average score cannot see the rare bad run it stops", + "seed": "GEVA-2, 2026-11-02: careful: exempt (safety guard)" + }, + "claude-code": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: claude-code: untested (never measured)" + }, + "codex": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "not in GEVA-2's table (2026-11-02); never measured" + }, + "context-restore": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: context-restore: untested (never measured)" + }, + "context-save": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: context-save: untested (never measured)" + }, + "cso": { + "status": "safety", + "grandfathered": true, + "reason": "security: supported security findings with explicit coverage; kept on its security reason (GEVA-2 marks it exempt; GStack Minder may reclassify it)", + "seed": "GEVA-2, 2026-11-02: cso: exempt (security audit skill, treated as exempt; GStack Minder may reclassify)" + }, + "design-consultation": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: design-consultation: untested (never measured)" + }, + "design-html": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: design-html: untested (never measured)" + }, + "design-motion": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: design-motion: untested (never measured)" + }, + "design-review": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-9 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: design-review: untested (never measured)" + }, + "design-shotgun": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: design-shotgun: untested (never measured)" + }, + "deslop-shared-libs": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: deslop-shared-libs: untested (never measured)" + }, + "devex-review": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: devex-review: untested (never measured)" + }, + "diagram": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: diagram: untested (never measured)" + }, + "document-generate": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: document-generate: untested (never measured)" + }, + "document-release": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: document-release: untested (never measured)" + }, + "eval-plan": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: eval-plan: untested (never measured)" + }, + "freeze": { + "status": "safety", + "grandfathered": true, + "reason": "safety guard: hard-blocks edits outside one directory; kept on that reason", + "seed": "GEVA-2, 2026-11-02: freeze: exempt (safety guard)" + }, + "gstack": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: gstack (root skill): untested (never measured)" + }, + "gstack-upgrade": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: upgrade: untested (never measured)" + }, + "guard": { + "status": "safety", + "grandfathered": true, + "reason": "safety guard: careful and freeze together; kept on that reason", + "seed": "GEVA-2, 2026-11-02: guard: exempt (safety guard)" + }, + "health": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: health: untested (never measured)" + }, + "investigate": { + "status": "untested", + "grandfathered": true, + "plan": "R3 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-101-r48-follow-ups-prereg (#112, main f0b81c0) and its amendment (open PR #133), told /investigate vs no gstack on SWE-bench Verified, preset, Opus 5.5 then GPT-6.1 Sol", + "prior": [ + { + "ref": "gstack-evals #109 stacks round 1 TB2 (GEVA-89, GEVA-93), wiki/board/2026-11-01-geva-93-stacks-round-1-tb2-results", + "why": [ + "stub", + "no-exposure", + "unmasked" + ], + "note": "Opus +3 points [-3, +9] at 1.9x cost; debug report in 0 of 111 trials; particle-1 -5 [-14, +1]" + }, + { + "ref": "GSTA-12 Lane B quick look 2, wiki/board/2026-10-29-gsta-12-laneb-quick-2-results", + "why": [ + "stub", + "unmasked" + ], + "note": "Opus gstack minus none -7 [-13, -2] at 1.7x cost; did not replicate" + }, + { + "ref": "GSTA-30 Lane B quick look 3 (#89), wiki/board/2026-10-30-gsta-30-laneb-quick-3-results", + "why": [ + "stub", + "small-tier" + ], + "note": "gstack arms on Sonnet, particle-1, DeepSeek and Haiku; Opus ran the none arm only" + }, + { + "ref": "HARN-14 replication (harness-evals #181, unmerged)", + "why": [ + "stub", + "small-tier" + ], + "note": "Haiku 5.5: gstack minus none +3.5 [-12.3, +20.0]" + } + ], + "r48": { + "verdict": "FIX-METHOD", + "ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R3 is the fix" + }, + "seed": "GEVA-2, 2026-11-02: investigate: measurement under way (R3)" + }, + "ios-clean": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ios-clean: untested (never measured)" + }, + "ios-design-review": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ios-design-review: untested (never measured)" + }, + "ios-fix": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ios-fix: untested (never measured)" + }, + "ios-qa": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ios-qa: untested (never measured)" + }, + "ios-sync": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ios-sync: untested (never measured)" + }, + "land-and-deploy": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: land-and-deploy: untested (never measured)" + }, + "landing-report": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: landing-report: untested (never measured)" + }, + "learn": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "prior": [ + { + "ref": "GSTA-13, GEVA-54 and GEVA-65 Lane A runs (wiki/board/2026-10-28-gsta-13-quick-look-3-results and the GEVA-54 and GEVA-65 posts)", + "why": [ + "stub", + "no-true-no-gstack-arm", + "ceiling" + ], + "note": "Opus contract tasks 27/30 vs 27/30" + }, + { + "ref": "GEVA-65 Lane A rerun row 5 (GPT-6.1 Sol in Codex)", + "why": [ + "not-claude-code", + "no-true-no-gstack-arm" + ], + "note": "Sol failed the no-code-change safety task in 5 of 9 skill trials vs 0 of 3 without; small n, a lead to check, not a hurts verdict" + } + ], + "seed": "GEVA-2, 2026-11-02: learn: untested (unfair test: SDK stub; no true no-gstack arm; Opus at ceiling; Sol ran in Codex, not Claude Code)" + }, + "make-pdf": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: make-pdf: untested (never measured)" + }, + "modernize": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: modernize: untested (never measured)" + }, + "office-hours": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "prior": [ + { + "ref": "GSTA-13 quick look 3 Lane A, wiki/board/2026-10-28-gsta-13-quick-look-3-results", + "why": [ + "stub", + "no-true-no-gstack-arm" + ], + "note": "contract tasks on Opus: gstack 74% vs none 28%; the tasks grade the skill's own contract, a manipulation check, not lift" + }, + { + "ref": "GEVA-65 Lane A rerun, wiki/board/2026-10-31-geva-65-lanea-shakedown-rerun-results", + "why": [ + "stub", + "no-true-no-gstack-arm" + ], + "note": "Opus before 0.67, after 0.76, none 0.36" + } + ], + "seed": "GEVA-2, 2026-11-02: office-hours: untested (unfair test: SDK stub; no true no-gstack arm; tasks grade the skill's own contract, a manipulation check, not lift)" + }, + "open-gstack-browser": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: open-gstack-browser: untested (never measured)" + }, + "pair-agent": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: pair-agent: untested (never measured)" + }, + "plan-ceo-review": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-5 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "prior": [ + { + "ref": "CEO panel v2 admission (GEVA-142), wiki/board/2026-11-02-geva-142-ceo-panel-v2-admission-results (open PR #131)", + "why": [ + "ceiling", + "unmasked" + ], + "note": "preset, Opus 5.5 only: must-fix recall 1.00 in all three arms; plain invocation ran the procedure in 0 of 16" + }, + { + "ref": "Planning R1a pilot (GEVA-102), wiki/board/2026-11-01-geva-102-planning-r1a-pilot-stop (#114)", + "why": [ + "stub", + "ceiling", + "unmasked" + ], + "note": "none-opus 8/8; gstack-opus read the sections file in 0 of 4" + } + ], + "r48": { + "verdict": "FIX-METHOD", + "ref": "CEO panel v2 admission (GEVA-142, open PR #131): FIX-METHOD on the panel, a harder panel is planned" + }, + "seed": "GEVA-2, 2026-11-02: plan-ceo-review: untested (unfair test: ceiling, control 12 of 12 so no MDD; plain invocation ran the procedure in 0 of 16)" + }, + "plan-design-review": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: plan-design-review: untested (never measured)" + }, + "plan-devex-review": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: plan-devex-review: untested (never measured)" + }, + "plan-eng-review": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: plan-eng-review: untested (never measured)" + }, + "plan-tune": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: plan-tune: untested (never measured)" + }, + "qa": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: qa: untested (never measured)" + }, + "qa-only": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: qa-only: untested (never measured)" + }, + "retro": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: retro: untested (never measured)" + }, + "review": { + "status": "untested", + "grandfathered": true, + "plan": "R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms", + "prior": [ + { + "ref": "gstack-evals #108 stacks round 1 review (GEVA-89), wiki/board/2026-11-01-geva-89-stacks-round-1-review-results", + "why": [ + "stub", + "no-exposure", + "unmasked" + ], + "note": "Opus 5.5 blocking recall 9.5% vs 7.1%, +2.4 points [-6.4, +11.1]; specialists and adversarial pass ran in 0 of 66 sessions" + }, + { + "ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results", + "why": [ + "stub", + "unmasked" + ], + "note": "gstack-opus minus none-opus +3.5 points [-9.6, +16.7]" + }, + { + "ref": "GSTA-11 quick look 1 review replay, wiki/board/2026-10-28-gsta-11-review-replay-results", + "why": [ + "not-claude-code", + "unmasked" + ], + "note": "/review's prompts through the Messages API; Opus recall 44% vs 39% plain, 12 bugs" + } + ], + "r48": { + "verdict": "FIX-METHOD", + "ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R2 is the fix" + }, + "seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)" + }, + "scrape": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: scrape: untested (never measured)" + }, + "setup-browser-cookies": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: setup-browser-cookies: untested (never measured)" + }, + "setup-deploy": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: setup-deploy: untested (never measured)" + }, + "setup-gbrain": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: setup-gbrain: untested (never measured)" + }, + "ship": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "seed": "GEVA-2, 2026-11-02: ship: untested (never measured)" + }, + "skillify": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: skillify: untested (never measured)" + }, + "spec": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: spec: untested (never measured)" + }, + "sync-gbrain": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: sync-gbrain: untested (never measured)" + }, + "test-audit": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: test-audit: untested (never measured)" + }, + "ubicloud": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: ubicloud: untested (never measured)" + }, + "unfreeze": { + "status": "safety", + "grandfathered": true, + "reason": "safety guard: the off switch for freeze's edit restriction; kept with the guard it releases", + "seed": "GEVA-2, 2026-11-02: unfreeze: exempt (safety guard (undoes freeze))" + }, + "verify": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "not in GEVA-2's table (2026-11-02); never measured" + }, + "write-skill": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: write-skill: untested (never measured)" + }, + "preamble": { + "status": "untested", + "grandfathered": true, + "plan": "I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1", + "prior": [ + { + "ref": "GEVA-100 exposure counts, org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100)", + "why": [ + "stub", + "no-exposure" + ], + "note": "ceremony sections ran in 0 of 66 Opus review sessions and 6 of 111 Opus TB2 trials; no effect estimate" + } + ], + "r48": { + "verdict": "CUT-CANDIDATE", + "ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); taken by Garry as the I1 ceremony-cut eval" + }, + "seed": "GEVA-2, 2026-11-02: preamble (ceremony): measurement under way (I1, ceremony-cut eval)" + }, + "catalog": { + "status": "untested", + "grandfathered": true, + "plan": "Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description", + "prior": [ + { + "ref": "GSTA-12 Lane B quick look 1, wiki/board/2026-10-29-gsta-12-laneb-quick-results", + "why": [ + "stub", + "no-exposure", + "unmasked" + ], + "note": "untold skill opened in 0 of 180 gstack-arm trials; Opus gstack minus none +5 [-3, +17]" + } + ], + "seed": "GEVA-2, 2026-11-02: always-on skill catalog (descriptions): measurement under way (Run D)" + }, + "capy-skill": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: capy/skill/gstack/SKILL.md (Capy context): untested (never measured)", + "note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it" + }, + "capy-factory": { + "status": "untested", + "grandfathered": true, + "plan": "none", + "seed": "GEVA-2, 2026-11-02: capy/skill/gstack/factory.md (Capy context): untested (never measured)", + "note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it" + }, + "ship-review-panel": { + "status": "untested", + "grandfathered": true, + "plan": "GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)", + "prior": [ + { + "ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results", + "why": [ + "stub", + "not-claude-code", + "unmasked" + ], + "note": "arm C minus none +56.9 points [40.5, 72.2]; the none arm ran on the stub, arm C in Capy at another gstack version, on a cohort mined from gstack's own catches" + }, + { + "ref": "GSTA-48 review no-loss arm D (#77), wiki/board/2026-10-30-gsta-48-review-noloss-arm-how-to", + "why": [ + "variant-only", + "not-claude-code" + ], + "note": "arm D vs pooled arm C, no no-gstack arm" + } + ], + "seed": "GEVA-2, 2026-11-02: ship: review panel (specialist, adversarial, targeted, outside voice, red team): untested (unfair test: none arm on the SDK stub; arms from different harnesses and gstack versions; cohort mined from gstack's own catches)" + }, + "ship-docs-audit": { + "status": "untested", + "grandfathered": true, + "plan": "folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered", + "seed": "GEVA-2, 2026-11-02: ship: docs audit (Step 14.5): untested (never measured)" + }, + "autoplan-phases": { + "status": "untested", + "grandfathered": true, + "plan": "folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered", + "seed": "GEVA-2, 2026-11-02: autoplan: CEO, design, DX and eng phases, outside voice: untested (never measured)" + } + }, + "unmapped": [ + { + "row": "review: Review Army and adversarial pass", + "why": "a section inside /review, not its own key; R2 measures it as part of the review row's plan" + }, + { + "row": "ship: QA probes (Step 9.2.1)", + "why": "a step inside /ship with no separate size; folded into the ship row (GEVA-2 registry row R56-4)" + }, + { + "row": "gstack as a whole (installed, any skill)", + "why": "not a single text part; GEVA-2's real-world outcome lane measures it (gstack-evals plan #111), not a measurement plan for any one part" + } + ] +} diff --git a/lib/inventory.ts b/lib/inventory.ts new file mode 100644 index 000000000..7e7dd7217 --- /dev/null +++ b/lib/inventory.ts @@ -0,0 +1,313 @@ +/** + * inventory — every part of gstack, its load per run and its evidence (ETHOS.md: 9. Earn Its Place). + * + * Keys are the skill directories discoverTemplates() finds (the root skill as `gstack`) plus + * SHARED_KEYS. The ledger, docs/inventory/evidence.json, is hand-kept: each row is checked by its + * status's own fields (validateEntry). buildInventory joins the ledger with lib/context-bill.ts + * buildBill (offline token estimates) and the dispatch counts ship and autoplan already document, + * each found by its quoted line so a citation that drifts is named. Read-only except the CLI's write. + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { discoverTemplates } from '../scripts/discover-skills'; +import { buildBill, contentClass, TOKEN_DIVISOR, TOKEN_DIVISORS } from './context-bill'; + +export const LEDGER_PATH = 'docs/inventory/evidence.json'; +export const DOC_PATH = 'docs/inventory.md'; +export const ROOT_KEY = 'gstack'; + +export interface SharedPart { key: string; label: string; load: string; files: string[] } +/** Shared parts: sized from their files (catalog from the bill's frontmatter). */ +export const SHARED_PARTS: SharedPart[] = [ + { key: 'preamble', label: 'shared preamble (sections/preamble.md, identical in every tier 2+ skill)', load: 'every tier 2+ skill run, inside that skill\'s load', files: ['ship/sections/preamble.md'] }, + { key: 'catalog', label: 'skill catalog (every skill\'s frontmatter)', load: 'every session', files: [] }, + { key: 'capy-skill', label: 'capy/skill/gstack/SKILL.md', load: 'every Capy session', files: ['capy/skill/gstack/SKILL.md'] }, + { key: 'capy-factory', label: 'capy/skill/gstack/factory.md', load: 'every Capy session', files: ['capy/skill/gstack/factory.md'] }, + { key: 'ship-review-panel', label: '/ship review panel (review-army and adversarial sections)', load: 'every /ship run, by depth', files: ['ship/sections/review-army.md', 'ship/sections/adversarial.md'] }, + { key: 'ship-docs-audit', label: '/ship docs audit (Step 14.5)', load: 'every /ship run', files: ['ship/sections/documentation.md'] }, + { key: 'autoplan-phases', label: '/autoplan CEO, design, DX and eng phases', load: 'every /autoplan run, by scope', files: ['autoplan/sections/ceo-phase.md', 'autoplan/sections/design-phase.md', 'autoplan/sections/dx-phase.md', 'autoplan/sections/eng-phase.md'] }, +]; +export const SHARED_KEYS = SHARED_PARTS.map(p => p.key); + +export interface Dispatch { skill: 'ship' | 'autoplan'; what: string; count: string; file: string; quote: string } +/** Paid dispatches per run, from the budget tables and docs that already state them (no new instrumentation). */ +export const DISPATCHES: Dispatch[] = [ + { skill: 'ship', what: 'Step 9 review, pass 1, light depth, docs or tests only', count: '1 reader (the core checklist)', file: 'docs/review-policy.md', quote: '| **light**, runtime 0 |' }, + { skill: 'ship', what: 'Step 9 review, pass 1, light depth, code', count: '3 readers (core, one batched specialist, native adversarial), plus the outside voice on a diff that publishes numbers', file: 'docs/review-policy.md', quote: '| **light**, runtime above 0 |' }, + { skill: 'ship', what: 'Step 9 review, pass 1, standard depth', count: 'the batched specialist, the Red Team at 200 runtime lines, adversarial and outside voice', file: 'docs/review-policy.md', quote: '| **standard** |' }, + { skill: 'ship', what: 'Step 9 review, pass 1, full depth', count: 'the batched specialist on the whole diff and the Red Team always', file: 'docs/review-policy.md', quote: '| **full** |' }, + { skill: 'ship', what: 'targeted reader on money, gate, auth or secret code', count: '+1 at every depth', file: 'docs/review-policy.md', quote: '`gstack-review-plan` dispatches one' }, + { skill: 'ship', what: 'audits (Steps 7, 8 and 14.5)', count: '3', file: 'lib/ship-budget-calibration.json', quote: 'each audit dispatch (Steps 7, 8, 14.5)' }, + { skill: 'ship', what: 'measured: one /ship child (gstack-evals#31)', count: '10 review dispatches', file: 'lib/ship-budget-calibration.json', quote: 'its 10 review dispatches' }, + { skill: 'ship', what: 'prior cost of one dispatch on the Capy meter', count: '7,430,000 tokens', file: 'lib/ship-budget-calibration.json', quote: '"per_dispatch": 7430000' }, + { skill: 'autoplan', what: 'voices per phase (native and outside)', count: '2', file: 'autoplan/DRIVER.md.tmpl', quote: 'per voice. Dispatch the native prompt to a subagent on your model and the outside prompt to' }, + { skill: 'autoplan', what: 'phases per round: CEO and Eng always, Design with UI scope, DX with developer-facing scope', count: '2 to 4, so 4 to 8 voice dispatches a round', file: 'autoplan/SKILL.md.tmpl', quote: 'Phases MUST run in strict order: CEO → Design (if UI scope) → DX (if developer-facing' }, + { skill: 'autoplan', what: 'CEO spec review loop', count: 'up to 3 launches', file: 'autoplan/sections/ceo-phase.md.tmpl', quote: 'three-launch cap' }, + { skill: 'autoplan', what: 'review rounds per phase (new unattended runs)', count: 'up to 2', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_MAX_ROUNDS = 2;' }, + { skill: 'autoplan', what: 'reserved per voice on Capy or under a spend cap', count: '$5', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_HOST_ESTIMATE_USD = 5;' }, +]; + +export const STATUSES = ['helps', 'no-effect', 'hurts', 'untested', 'safety'] as const; +export type Status = (typeof STATUSES)[number]; +export const R48_VERDICTS = ['KEEP', 'FIX-METHOD', 'CUT-CANDIDATE'] as const; +/** R54: the frontier models under test; a row may add at most one named low-cost primary, never Haiku or Sonnet. */ +export const FRONTIER_MODELS = ['claude-opus-5-5', 'gpt-6.1-sol']; +/** Why an earlier result does not count as fair evidence yet. */ +export const PRIOR_WHY: Record = { + stub: 'ran on the Agent SDK stub prompt, not the claude_code preset', + 'small-tier': 'Haiku, Sonnet or another non-frontier model only', + unmasked: 'no mask recorded: the no-gstack arm predates the masking rule and was not re-checked', + ceiling: 'control at the ceiling, so no difference could show', + 'no-true-no-gstack-arm': 'the none arm still had gstack installed', + 'no-exposure': 'the model never used the part', + 'not-claude-code': 'ran outside Claude Code (Messages API, Codex or Capy)', + 'variant-only': 'variant against variant, no no-gstack arm', +}; +const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'seed', 'note']; + +export type LedgerCode = + | 'LEDGER_PARSE' | 'LEDGER_KEY_MISSING' | 'LEDGER_KEY_EXTRA' | 'LEDGER_STATUS_INVALID' | 'LEDGER_FIELD_UNKNOWN' + | 'LEDGER_SOURCES_MISSING' | 'LEDGER_MODELS_NOT_FRONTIER' | 'LEDGER_PROMPT_NOT_PRESET' | 'LEDGER_SCOPE_MISSING' + | 'LEDGER_UNMASKED' | 'LEDGER_R48_MISSING' | 'LEDGER_PLAN_MISSING' | 'LEDGER_PLAN_NONE_NEW' | 'LEDGER_PLAN_NOT_CONCRETE' + | 'LEDGER_REASON_MISSING' | 'LEDGER_OWNER_MISSING' | 'LEDGER_MEASURE_BY_MISSING' | 'LEDGER_PRIOR_INVALID' + | 'INVENTORY_SOURCE_MISSING' | 'INVENTORY_PART_MISSING'; +export interface LedgerProblem { key: string; code: LedgerCode; message: string } +export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; seed?: string; note?: string } +export interface Ledger { parts: Record; unmapped?: Array<{ row: string; why: string }>; [k: string]: unknown } + +const isObj = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const str = (v: unknown): v is string => typeof v === 'string' && v.trim() !== ''; +const validR48 = (v: unknown) => isObj(v) && (R48_VERDICTS as readonly string[]).includes(v.verdict as string) && str(v.ref); + +function frontierModels(v: unknown): boolean { + if (!Array.isArray(v) || !v.length || !v.every(str)) return false; + const ids = v.map(m => m.slice(m.lastIndexOf('/') + 1)); + const other = ids.filter(m => !FRONTIER_MODELS.includes(m)); + return ids.length > other.length && other.length <= 1 && !other.some(m => /haiku|sonnet/i.test(m)); +} + +/** Problems with one ledger row, by its status's own fields (helps, no-effect, hurts, untested, safety). */ +export function validateEntry(key: string, entry: unknown): LedgerProblem[] { + const problems: LedgerProblem[] = []; + const add = (code: LedgerCode, message: string) => problems.push({ key, code, message }); + if (!isObj(entry)) { add('LEDGER_STATUS_INVALID', 'the row is not an object'); return problems; } + for (const f of Object.keys(entry)) if (!FIELDS.includes(f)) add('LEDGER_FIELD_UNKNOWN', `unknown field ${f}; fields: ${FIELDS.join(', ')}`); + const prior = entry.prior; + if (prior !== undefined && !(Array.isArray(prior) && prior.every(p => isObj(p) && str(p.ref) && Array.isArray(p.why) && p.why.length > 0 && p.why.every(w => typeof w === 'string' && w in PRIOR_WHY)))) + add('LEDGER_PRIOR_INVALID', `prior needs [{ref, why: [...]}] with why from: ${Object.keys(PRIOR_WHY).join(', ')}`); + const status = entry.status; + if (status === 'helps' || status === 'no-effect' || status === 'hurts') { + if (!Array.isArray(entry.sources) || !entry.sources.length || !entry.sources.every(str)) add('LEDGER_SOURCES_MISSING', `${status} needs sources: one or more run ids or results files`); + if (!frontierModels(entry.models)) add('LEDGER_MODELS_NOT_FRONTIER', `models must include ${FRONTIER_MODELS.join(' or ')} plus at most one named low-cost primary, never Haiku or Sonnet (R54)`); + if (entry.prompt !== 'preset') add('LEDGER_PROMPT_NOT_PRESET', 'prompt must be "preset" (the claude_code system prompt, not the Agent SDK stub)'); + if (!str(entry.scope)) add('LEDGER_SCOPE_MISSING', 'scope must say what was measured'); + if (entry.masked !== true) add('LEDGER_UNMASKED', 'masked must be true: the no-gstack arm was masked from the Capy drives and gstack installs'); + if (status !== 'helps' && !validR48(entry.r48)) add('LEDGER_R48_MISSING', `${status} needs r48: {verdict: ${R48_VERDICTS.join('|')}, ref}`); + } else if (status === 'untested') { + const grandfathered = entry.grandfathered === true; + if (!str(entry.plan)) add('LEDGER_PLAN_MISSING', 'untested needs plan: a reference, or "none" on a grandfathered row'); + else if (entry.plan === 'none') { if (!grandfathered) add('LEDGER_PLAN_NONE_NEW', 'plan: none is only for grandfathered rows; a new part names its measurement plan'); } + else if (!grandfathered && !/tasks\.json$|prereg/i.test(entry.plan)) add('LEDGER_PLAN_NOT_CONCRETE', 'a new part\'s plan is a test/fixtures/skill-evals//tasks.json or an /eval-plan preregistration'); + if (!grandfathered) { + if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'a new untested part states its reason'); + if (!str(entry.owner)) add('LEDGER_OWNER_MISSING', 'a new untested part names its owner'); + if (!str(entry.measure_by) || !/^\d{4}-\d{2}-\d{2}$/.test(entry.measure_by) || Number.isNaN(Date.parse(entry.measure_by))) add('LEDGER_MEASURE_BY_MISSING', 'a new untested part needs measure_by: YYYY-MM-DD'); + } + } else if (status === 'safety') { + if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'safety needs reason: what the guard protects'); + } else add('LEDGER_STATUS_INVALID', `status must be one of ${STATUSES.join(', ')}`); + if (entry.r48 !== undefined && !validR48(entry.r48) && (status === 'untested' || status === 'safety')) add('LEDGER_R48_MISSING', `r48 needs {verdict: ${R48_VERDICTS.join('|')}, ref}`); + return problems; +} + +/** Coverage (no missing, no extra key) then every row's own schema. */ +export function validateLedger(ledger: unknown, keys: string[]): LedgerProblem[] { + if (!isObj(ledger) || !isObj(ledger.parts)) return [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `${LEDGER_PATH} needs a parts object` }]; + const parts = ledger.parts; + return [ + ...keys.filter(k => !(k in parts)).map(k => ({ key: k, code: 'LEDGER_KEY_MISSING' as const, message: `no row in ${LEDGER_PATH}` })), + ...Object.keys(parts).filter(k => !keys.includes(k)).map(k => ({ key: k, code: 'LEDGER_KEY_EXTRA' as const, message: 'not an inventory key (the key list is in docs/inventory.md)' })), + ...keys.filter(k => k in parts).flatMap(k => validateEntry(k, parts[k])), + ]; +} + +export function loadLedger(root: string): { ledger: Ledger | null; problems: LedgerProblem[] } { + try { + return { ledger: JSON.parse(fs.readFileSync(path.join(root, LEDGER_PATH), 'utf8')), problems: [] }; + } catch (e) { + return { ledger: null, problems: [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `cannot read ${LEDGER_PATH}: ${(e as Error).message}` }] }; + } +} + +/** The skill keys (discoverTemplates, root skill as gstack, symlinked folders excluded) then SHARED_KEYS. */ +export function inventoryKeys(root: string): string[] { + const skills = discoverTemplates(root).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d)); + return [...skills.sort(), ...SHARED_KEYS]; +} + +/** Installed skill name to key: gstack- is ; gstack-upgrade keeps its name. */ +export const keyOf = (name: string) => (name !== 'gstack-upgrade' && name.startsWith('gstack-') ? name.slice('gstack-'.length) : name); + +/** The status's evidence in one phrase: sources, plan or reason. */ +export function evidenceRef(e: LedgerEntry): string { + if (e.status === 'untested') return `plan: ${e.plan}${e.grandfathered ? ' (grandfathered)' : ` (owner ${e.owner}, measure by ${e.measure_by})`}`; + if (e.status === 'safety') return `reason: ${e.reason}`; + return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}`; +} + +/** gstack-skill-bill's EVIDENCE line for one skill folder in a gstack checkout. */ +export function skillEvidence(root: string, dir: string): { key: string; status: Status | 'missing'; ref: string; line: string } { + const key = path.relative(root, dir) || ROOT_KEY; + const entry = loadLedger(root).ledger?.parts?.[key]; + if (!entry) { + const ref = `no ${key} row in ${LEDGER_PATH}; add one (ETHOS.md: 9. Earn Its Place)`; + return { key, status: 'missing', ref, line: `EVIDENCE: missing: ${ref}` }; + } + const ref = evidenceRef(entry); + return { key, status: entry.status, ref, line: `EVIDENCE: ${entry.status} ${ref}` }; +} + +export interface CitedDispatch extends Dispatch { line: number | null } +const tokensOf = (rel: string, bytes: number) => Math.round(bytes / (TOKEN_DIVISORS[contentClass(rel)] ?? TOKEN_DIVISOR)); + +/** Ledger, dispatch citations and shared-part files; no bill, so --check stays fast. */ +export function checkInventory(root: string): { keys: string[]; ledger: Ledger | null; dispatches: CitedDispatch[]; problems: LedgerProblem[] } { + const keys = inventoryKeys(root); + const loaded = loadLedger(root); + const problems = [...loaded.problems, ...(loaded.ledger ? validateLedger(loaded.ledger, keys) : [])]; + const dispatches = DISPATCHES.map(d => { + const file = path.join(root, d.file); + const idx = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').findIndex(l => l.includes(d.quote)) : -1; + if (idx < 0) problems.push({ key: `dispatch:${d.skill}`, code: 'INVENTORY_SOURCE_MISSING', message: `${d.file} no longer holds "${d.quote}"; update DISPATCHES in lib/inventory.ts` }); + return { ...d, line: idx < 0 ? null : idx + 1 }; + }); + for (const part of SHARED_PARTS) for (const f of part.files) if (!fs.existsSync(path.join(root, f))) + problems.push({ key: part.key, code: 'INVENTORY_PART_MISSING', message: `${f} is missing; update SHARED_PARTS in lib/inventory.ts` }); + return { keys, ledger: loaded.ledger, dispatches, problems }; +} + +export interface PartRow { key: string; kind: 'skill' | 'shared'; label: string; loads: string; loadTokens: number | null; alwaysOnTokens: number | null; status: Status | 'missing'; ref: string } +export interface Inventory { + commit: string; date: string; tokenSource: string; keys: string[]; parts: PartRow[]; + tiers: Array<{ tier: string; skills: string[] }>; dispatches: CitedDispatch[]; + prior: Array<{ key: string; ref: string; why: string[]; note?: string }>; + unmapped: Array<{ row: string; why: string }>; problems: LedgerProblem[]; +} + +function git(root: string, args: string[]): string | null { + const r = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8', timeout: 10_000 }); + return r.status === 0 ? r.stdout.trim() : null; +} + +/** The commit the snapshot measured, its date, and whether the tree had other changes. */ +function measuredAt(root: string): { commit: string; date: string } { + const top = git(root, ['rev-parse', '--show-toplevel']); + if (!top || path.resolve(top) !== path.resolve(root)) return { commit: 'none', date: 'none' }; + const dirty = git(root, ['status', '--porcelain', '--', '.', `:!${DOC_PATH}`]); + return { commit: (git(root, ['rev-parse', '--short=9', 'HEAD']) ?? 'none') + (dirty ? ' (with uncommitted changes)' : ''), date: git(root, ['log', '-1', '--format=%cs']) ?? 'none' }; +} + +/** Joins the ledger with the context bill: one row per key, ranked by load per run. */ +export function buildInventory(root: string): Inventory { + const checked = checkInventory(root); + const bill = buildBill(root); + const resolved = path.resolve(root); + const bySkill = new Map(bill.skills.map(s => [path.relative(resolved, s.dir) || ROOT_KEY, s])); + const parts = checked.ledger?.parts ?? {}; + const skillKeys = checked.keys.filter(k => !SHARED_KEYS.includes(k)); + const evidence = (key: string) => (parts[key] ? { status: parts[key]!.status, ref: evidenceRef(parts[key]!) } : { status: 'missing' as const, ref: 'no row' }); + const rows: PartRow[] = skillKeys.map(key => { + const s = bySkill.get(key); + return { key, kind: 'skill', label: key === ROOT_KEY ? 'root skill (/gstack router)' : `/${key}`, loads: 'every use', loadTokens: s ? Math.round(s.perInvocationTokens) : null, alwaysOnTokens: s ? Math.round(s.frontmatterTokens) : null, ...evidence(key) }; + }); + const catalog = skillKeys.reduce((n, k) => n + (bySkill.get(k)?.frontmatterTokens ?? 0), 0); + for (const part of SHARED_PARTS) { + const sizes = part.files.map(f => { const file = path.join(root, f); return fs.existsSync(file) ? tokensOf(f, fs.statSync(file).size) : 0; }); + const tokens = part.key === 'catalog' ? Math.round(catalog) : sizes.reduce((a, b) => a + b, 0); + rows.push({ key: part.key, kind: 'shared', label: part.label, loads: part.load, loadTokens: tokens, alwaysOnTokens: part.key === 'catalog' ? tokens : null, ...evidence(part.key) }); + } + rows.sort((a, b) => (b.loadTokens ?? -1) - (a.loadTokens ?? -1) || a.key.localeCompare(b.key)); + const tiers = new Map(); + for (const key of skillKeys) { + const tmpl = path.join(root, key === ROOT_KEY ? '' : key, 'SKILL.md.tmpl'); + const tier = /^preamble-tier:\s*(\d+)/m.exec(fs.readFileSync(tmpl, 'utf8'))?.[1] ?? 'none'; + tiers.set(tier, [...(tiers.get(tier) ?? []), key]); + } + return { + ...measuredAt(root), + tokenSource: bill.tokenSource, + keys: checked.keys, + parts: rows, + tiers: [...tiers].sort(([a], [b]) => a.localeCompare(b)).map(([tier, skills]) => ({ tier, skills })), + dispatches: checked.dispatches, + prior: checked.keys.flatMap(key => (parts[key]?.prior ?? []).map(p => ({ key, ...p }))), + unmapped: checked.ledger?.unmapped ?? [], + problems: checked.problems, + }; +} + +const n = (v: number | null) => (v === null ? 'n/a' : v.toLocaleString('en-US')); +const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/\n/g, ' '); + +function renderRanking(inv: Inventory): string[] { + const out = ['## Parts by load per run', '', 'Load per run: a skill\'s SKILL.md plus the files it must read on every use (`perInvocationTokens` in lib/context-bill.ts); a shared part\'s files. Always-on: frontmatter every session\'s skill scanner reads.', '', + '| Rank | Part | Kind | Load per run (tokens) | Always-on (tokens) | Loads | Status | Source, plan or reason |', '|---|---|---|---|---|---|---|---|']; + inv.parts.forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${p.kind} | ${n(p.loadTokens)} | ${n(p.alwaysOnTokens)} | ${cell(p.loads)} | ${p.status} | ${cell(p.ref)} |`)); + return out; +} + +function renderMeasureNext(inv: Inventory): string[] { + const out = ['## Measure next', '', 'Untested parts, largest load per run first. A fair test runs the claude_code preset, frontier models (Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary) and a masked no-gstack arm ([TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm)). Safety rows are not on this list.', '', + '| Order | Part | Load per run (tokens) | Plan |', '|---|---|---|---|']; + inv.parts.filter(p => p.status === 'untested').forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${n(p.loadTokens)} | ${cell(p.ref.replace(/^plan: /, ''))} |`)); + return out; +} + +function renderShared(inv: Inventory): string[] { + const out = ['## Shared parts and preamble tiers', '', '| Tier | Skills | What loads |', '|---|---|---|']; + const what: Record = { '1': 'a short preamble inline in SKILL.md', none: 'no preamble' }; + for (const t of inv.tiers) out.push(`| ${t.tier} | ${t.skills.join(', ')} | ${what[t.tier] ?? 'the shared sections/preamble.md (row preamble), once per session'} |`); + out.push('', ...SHARED_PARTS.map(p => `- **${p.key}**: ${p.label}${p.files.length ? ` (${p.files.join(', ')})` : ''}; loads ${p.load}.`)); + return out; +} + +function renderDispatches(inv: Inventory): string[] { + const out = ['## Paid dispatches per run', '', 'Each count is quoted from the file and line named; `bin/gstack-inventory --check` fails when a quoted line moves away.', '', '| Skill | What | Count | Source |', '|---|---|---|---|']; + for (const d of inv.dispatches) out.push(`| ${d.skill} | ${cell(d.what)} | ${cell(d.count)} | ${d.line ? `${d.file}:${d.line}` : `${d.file}: source not found`} |`); + return out; +} + +function renderPrior(inv: Inventory): string[] { + const out = ['## Earlier results that do not count yet', '', ...Object.entries(PRIOR_WHY).map(([k, v]) => `- \`${k}\`: ${v}.`), '', '| Part | Result | Why it does not count | Note |', '|---|---|---|---|']; + for (const p of inv.prior) out.push(`| ${p.key} | ${cell(p.ref)} | ${p.why.join(', ')} | ${cell(p.note ?? '')} |`); + return out; +} + +function renderKeys(inv: Inventory): string[] { + const skills = inv.keys.filter(k => !SHARED_KEYS.includes(k)); + return ['## Key list and aliases', '', `${inv.keys.length} keys: ${skills.length} skills (the folders \`discoverTemplates()\` finds, the root skill as \`gstack\`) and ${SHARED_KEYS.length} shared parts.`, '', + `- Skills: ${skills.join(', ')}.`, `- Shared: ${SHARED_KEYS.join(', ')}.`, + '- Installed names: `gstack-` is key `` (`gstack-ship` is `ship`); `gstack-upgrade` keeps its name; the root skill installs as `gstack`.']; +} + +function renderNotHere(inv: Inventory): string[] { + return ['## Not in this inventory', '', + '- Out of scope: openclaw/skills/* and browser-skills/* (not gstack skills an agent loads from this checkout), and connect-chrome (a symlink to open-gstack-browser, counted once).', + ...inv.unmapped.map(u => `- ${u.row}: ${u.why}`)]; +} + +export function renderInventory(inv: Inventory): string { + const count = (s: string) => inv.parts.filter(p => p.status === s).length; + const planned = inv.parts.filter(p => p.status === 'untested' && !p.ref.startsWith('plan: none')).length; + return [ + '# gstack inventory: every part, its load per run and its evidence', '', + `Generated by \`bin/gstack-inventory\` from [evidence.json](inventory/evidence.json), the hand-kept ledger, and lib/context-bill.ts. Measured at commit ${inv.commit} (${inv.date}). This is a dated snapshot: rerun \`bin/gstack-inventory\` to refresh it; \`bin/gstack-inventory --check\` validates the ledger. The rule: [ETHOS.md: 9. Earn Its Place](../ETHOS.md#9-earn-its-place).`, '', + `Token counts are offline estimates (${inv.tokenSource}, about ±40%), not billed tokens.`, '', + '| Status | Parts |', '|---|---|', + ...(['helps', 'no-effect', 'hurts'] as const).map(s => `| ${s} | ${count(s)} |`), + `| untested, with a plan | ${planned} |`, `| untested, plan none | ${count('untested') - planned} |`, `| safety | ${count('safety')} |`, '', + ...renderRanking(inv), '', ...renderMeasureNext(inv), '', ...renderShared(inv), '', ...renderDispatches(inv), '', + ...renderPrior(inv), '', ...renderKeys(inv), '', ...renderNotHere(inv), '', + ].join('\n'); +} diff --git a/lib/skill-bill.ts b/lib/skill-bill.ts index aeca2a739..c2e72590d 100644 --- a/lib/skill-bill.ts +++ b/lib/skill-bill.ts @@ -11,6 +11,7 @@ import { homedir } from "node:os"; import { basename, dirname, join, relative, resolve } from "node:path"; import { spawnSync } from "node:child_process"; import { estimateTokens } from "./context-bill"; +import { skillEvidence } from "./inventory"; export const DUE_PCT = 10; @@ -28,6 +29,8 @@ export interface SkillBillReport { rows: LevelRow[]; baseline: Baseline; evalTasks?: EvalTasks | null; + /** gstack checkout only: the skill's row in docs/inventory/evidence.json (ETHOS.md: 9. Earn Its Place). */ + evidence?: ReturnType; warnings: string[]; } @@ -196,8 +199,10 @@ export function buildSkillBill(dir: string): SkillBillReport { for (const f of onDemandFiles(dir)) rows.push(row("on-demand", dir, f)); const warnings: string[] = []; let evalTasks: EvalTasks | null | undefined; + let evidence: SkillBillReport["evidence"]; if (kind === "gstack-checkout") { const root = gstackRootOf(dir)!; + evidence = skillEvidence(root, dir); const file = join(root, "test", "fixtures", "skill-evals", name, "tasks.json"); evalTasks = existsSync(file) ? { ...checkEvalTasks(file), path: `test/fixtures/skill-evals/${name}/tasks.json` } : null; if (evalTasks?.problems.length) warnings.push(`EVAL_TASKS_INVALID: ${evalTasks.problems[0]}${evalTasks.problems.length > 1 ? ` (+${evalTasks.problems.length - 1} more)` : ""}. fix: the task format in docs/designs/SKILL_EVAL_CONTRACT.md`); @@ -206,7 +211,7 @@ export function buildSkillBill(dir: string): SkillBillReport { warnings.push("RENDER_STALE: SKILL.md.tmpl changed but SKILL.md did not. fix: bun run gen:skill-docs"); } if (kind === "gstack-install") warnings.push("INSTALLED_GSTACK_SKILL: gstack-upgrade overwrites this folder. fix: edit /SKILL.md.tmpl in a gstack checkout, or copy the skill to a folder you own"); - return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, warnings }; + return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, evidence, warnings }; } export function renderSkillBill(r: SkillBillReport): string { @@ -217,6 +222,7 @@ export function renderSkillBill(r: SkillBillReport): string { else lines.push(`BASELINE: ${b.status} (${b.reason})`); if (r.evalTasks) lines.push(`EVAL TASKS: ${r.evalTasks.path} (${r.evalTasks.count} tasks)`); else if (r.evalTasks === null) lines.push(`EVAL TASKS: none; a diet writes test/fixtures/skill-evals/${r.name}/tasks.json before any cut`); + if (r.evidence) lines.push(r.evidence.line); lines.push(...r.warnings); lines.push(`Commit line: Skill-Size: → bytes (${r.authored})`); return lines.join("\n"); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index ad2cf1081..a11b5d027 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -37,6 +37,7 @@ { "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" }, { "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" }, { "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" }, + { "file": "lib/inventory.ts", "movedFrom": "(new: the R56 evidence ledger check and the inventory behind bin/gstack-inventory, GST-914)" }, { "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" }, { "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" }, { "file": "lib/owner-brief.ts", "movedFrom": "(new: the one-page owner brief and draft-direction close behind bin/gstack-owner-brief, plan E3)" }, diff --git a/test/inventory.test.ts b/test/inventory.test.ts new file mode 100644 index 000000000..6ef746820 --- /dev/null +++ b/test/inventory.test.ts @@ -0,0 +1,188 @@ +/** + * R56 (ETHOS.md: 9. Earn Its Place): every part of gstack has a row in + * docs/inventory/evidence.json, validated by its status's own fields, and + * bin/gstack-inventory ranks the parts by load per run into docs/inventory.md. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { discoverTemplates } from '../scripts/discover-skills'; +import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin', 'gstack-inventory'); +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-inventory-')); +afterAll(() => fs.rmSync(tmp, { recursive: true, force: true })); + +const codes = (key: string, entry: unknown) => validateEntry(key, entry).map(p => p.code); +const run = (args: string[]) => spawnSync(BIN, args, { cwd: ROOT, encoding: 'utf8', timeout: 120_000 }); + +const VERDICT = { sources: ['gstack-evals run r9 results.json'], models: ['claude-opus-5-5', 'gpt-6.1-sol'], prompt: 'preset', scope: 'the full procedure, told', masked: true }; +const NEW_UNTESTED = { status: 'untested', reason: 'catches missed migrations', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-168', measure_by: '2026-12-01' }; + +describe('the key list', () => { + test('keys are the skill directories discoverTemplates finds, the root skill as gstack, then the fixed shared keys', () => { + const keys = inventoryKeys(ROOT); + const skills = discoverTemplates(ROOT).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d)); + expect(keys).toEqual([...skills.sort(), ...SHARED_KEYS]); + expect(keys).toContain('gstack'); + expect(keys).toContain('gstack-upgrade'); + expect(keys).not.toContain('connect-chrome'); + expect(new Set(keys).size).toBe(keys.length); + }); + + test('installed names map to keys: gstack- to , gstack-upgrade stays literal', () => { + expect(keyOf('gstack-ship')).toBe('ship'); + expect(keyOf('ship')).toBe('ship'); + expect(keyOf('gstack-upgrade')).toBe('gstack-upgrade'); + expect(keyOf('gstack')).toBe('gstack'); + }); +}); + +describe('the ledger covers every key and validates', () => { + const ledger = JSON.parse(fs.readFileSync(path.join(ROOT, LEDGER_PATH), 'utf8')); + + test('no missing and no extra keys, and every row passes its status schema', () => { + const keys = inventoryKeys(ROOT); + expect(Object.keys(ledger.parts).filter(k => !keys.includes(k))).toEqual([]); + expect(keys.filter(k => !(k in ledger.parts))).toEqual([]); + expect(validateLedger(ledger, keys)).toEqual([]); + }); + + test('the initial backlog is grandfathered and the guards are safety rows', () => { + for (const [key, row] of Object.entries(ledger.parts)) expect([key, row.grandfathered]).toEqual([key, true]); + for (const key of ['careful', 'freeze', 'guard', 'unfreeze', 'cso']) expect(ledger.parts[key].status).toBe('safety'); + for (const key of ['capy-skill', 'capy-factory']) expect(ledger.parts[key]).toMatchObject({ status: 'untested', plan: 'none' }); + }); + + test('a missing and an extra key are each named', () => { + const keys = inventoryKeys(ROOT); + const { ship, ...rest } = ledger.parts; + const problems = validateLedger({ ...ledger, parts: { ...rest, 'no-such-part': ship } }, keys); + expect(problems.map(p => `${p.key} ${p.code}`)).toEqual(['ship LEDGER_KEY_MISSING', 'no-such-part LEDGER_KEY_EXTRA']); + }); +}); + +describe('the per-status schema', () => { + test('passing rows: a grandfathered untested part with plan none, a new part with a dated plan, a fair verdict, a safety guard', () => { + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none' })).toEqual([]); + expect(codes('a', NEW_UNTESTED)).toEqual([]); + expect(codes('a', { ...NEW_UNTESTED, plan: 'docs/evals/demo/PREREG.md' })).toEqual([]); + expect(codes('a', { status: 'helps', ...VERDICT })).toEqual([]); + expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1'] })).toEqual([]); + expect(codes('a', { status: 'no-effect', ...VERDICT, r48: { verdict: 'CUT-CANDIDATE', ref: 'why-null.md' } })).toEqual([]); + expect(codes('a', { status: 'safety', reason: 'blocks rm -rf' })).toEqual([]); + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['stub', 'unmasked'] }] })).toEqual([]); + }); + + test('a new untested part needs reason, a concrete plan, owner and measure_by; plan none is refused', () => { + const { measure_by, ...noDate } = NEW_UNTESTED; + expect(codes('a', noDate)).toEqual(['LEDGER_MEASURE_BY_MISSING']); + expect(codes('a', { ...NEW_UNTESTED, measure_by: 'soon' })).toEqual(['LEDGER_MEASURE_BY_MISSING']); + const { owner, ...noOwner } = NEW_UNTESTED; + expect(codes('a', noOwner)).toEqual(['LEDGER_OWNER_MISSING']); + const { reason, ...noReason } = NEW_UNTESTED; + expect(codes('a', noReason)).toEqual(['LEDGER_REASON_MISSING']); + expect(codes('a', { ...NEW_UNTESTED, plan: 'none' })).toEqual(['LEDGER_PLAN_NONE_NEW']); + expect(codes('a', { ...NEW_UNTESTED, plan: 'we will test it later' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']); + expect(codes('a', { status: 'untested', grandfathered: true })).toEqual(['LEDGER_PLAN_MISSING']); + }); + + test('helps, no-effect and hurts need sources, frontier models, the preset prompt, a scope and a masked no-gstack arm', () => { + const { sources, ...noSources } = VERDICT; + expect(codes('a', { status: 'helps', ...noSources })).toEqual(['LEDGER_SOURCES_MISSING']); + expect(codes('a', { status: 'helps', ...VERDICT, sources: [] })).toEqual(['LEDGER_SOURCES_MISSING']); + expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-haiku-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']); + expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'claude-sonnet-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']); + expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1', 'deepseek-v4.1-flash'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']); + expect(codes('a', { status: 'helps', ...VERDICT, prompt: 'sdk-stub' })).toEqual(['LEDGER_PROMPT_NOT_PRESET']); + expect(codes('a', { status: 'hurts', ...VERDICT, scope: '', r48: { verdict: 'KEEP', ref: 'x' } })).toEqual(['LEDGER_SCOPE_MISSING']); + expect(codes('a', { status: 'helps', ...VERDICT, masked: false })).toEqual(['LEDGER_UNMASKED']); + }); + + test('no-effect and hurts without an R48 verdict and its reference are refused', () => { + expect(codes('a', { status: 'no-effect', ...VERDICT })).toEqual(['LEDGER_R48_MISSING']); + expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'DROP', ref: 'x' } })).toEqual(['LEDGER_R48_MISSING']); + expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'KEEP' } })).toEqual(['LEDGER_R48_MISSING']); + }); + + test('safety needs a reason; unknown statuses, unknown fields and bad prior entries are refused', () => { + expect(codes('a', { status: 'safety' })).toEqual(['LEDGER_REASON_MISSING']); + expect(codes('a', { status: 'exempt', reason: 'x' })).toEqual(['LEDGER_STATUS_INVALID']); + expect(codes('a', { status: 'safety', reason: 'x', source: 'typo' })).toEqual(['LEDGER_FIELD_UNKNOWN']); + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['vibes'] }] })).toEqual(['LEDGER_PRIOR_INVALID']); + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ why: ['stub'] }] })).toEqual(['LEDGER_PRIOR_INVALID']); + }); +}); + +/** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */ +function fixtureTree(dir: string, parts: (keys: string[]) => Record): string { + const write = (rel: string, text: string) => { fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); fs.writeFileSync(path.join(dir, rel), text); }; + const skill = (rel: string, name: string, tier: number, body: string) => { + const fm = `---\nname: ${name}\npreamble-tier: ${tier}\ndescription: |\n Does ${name} things.\n---\n`; + write(path.join(rel, 'SKILL.md.tmpl'), fm + body); + write(path.join(rel, 'SKILL.md'), fm + '\n' + body); + }; + skill('', 'gstack', 1, 'router\n'); + skill('alpha', 'alpha', 2, 'a'.repeat(4000) + '\n'); + skill('beta', 'beta', 2, 'b'.repeat(1000) + '\n'); + write('alpha/sections/preamble.md', 'p'.repeat(800)); + write('beta/sections/preamble.md', 'p'.repeat(800)); + for (const part of SHARED_PARTS) for (const f of part.files) write(f, 'x'.repeat(500) + '\n'); + for (const d of DISPATCHES) if (!fs.existsSync(path.join(dir, d.file))) write(d.file, 'filler\n'); + for (const d of DISPATCHES) fs.appendFileSync(path.join(dir, d.file), `${d.quote}\n`); + const keys = inventoryKeys(dir); + write(LEDGER_PATH, JSON.stringify({ schema_version: 1, parts: parts(keys), unmapped: [{ row: 'gstack as a whole', why: 'not a single text part' }] }, null, 2)); + return dir; +} + +const grandfathered = (keys: string[]) => Object.fromEntries(keys.map(k => [k, k === 'beta' ? { status: 'safety', grandfathered: true, reason: 'a guard' } : { status: 'untested', grandfathered: true, plan: 'none' }])); + +describe('bin/gstack-inventory', () => { + test('determinism: two runs on a fixed tree write the same docs/inventory.md and the same --json', () => { + const dir = fixtureTree(path.join(tmp, 'fixed'), grandfathered); + const first = run(['--root', dir]); + expect(first.status, first.stderr).toBe(0); + const a = fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8'); + const second = run(['--root', dir]); + expect(second.status, second.stderr).toBe(0); + expect(fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8')).toBe(a); + expect(run(['--root', dir, '--json']).stdout).toBe(run(['--root', dir, '--json']).stdout); + expect(a).toContain('Measured at commit none'); + expect(a.indexOf('| alpha |')).toBeLessThan(a.indexOf('| beta |')); + const start = a.indexOf('## Measure next'); + const next = a.slice(start, a.indexOf('\n## ', start + 1)); + expect(next).toContain('alpha'); + expect(next).not.toContain('| beta |'); + expect(a).toContain('gstack as a whole'); + expect(a).toContain('gstack-'); + for (const d of DISPATCHES) expect(a).toContain(`${d.file}:`); + }); + + test('--check: ok on a complete ledger; a missing key exits 1 with its code', () => { + const dir = fixtureTree(path.join(tmp, 'check'), grandfathered); + const ok = run(['--root', dir, '--check']); + expect(ok.status, ok.stderr).toBe(0); + expect(ok.stdout).toMatch(/^INVENTORY: ok keys=\d+/m); + const bad = fixtureTree(path.join(tmp, 'check-bad'), keys => { const p = grandfathered(keys); delete p.alpha; return p; }); + const r = run(['--root', bad, '--check']); + expect(r.status).toBe(1); + expect(r.stderr).toContain('alpha: no row in docs/inventory/evidence.json (LEDGER_KEY_MISSING)'); + expect(run(['--root', bad]).status).toBe(1); + }); + + test('--check: a cited dispatch source that lost its quote is named', () => { + const dir = fixtureTree(path.join(tmp, 'drift'), grandfathered); + fs.writeFileSync(path.join(dir, DISPATCHES[0]!.file), 'rewritten\n'); + const r = run(['--root', dir, '--check']); + expect(r.status).toBe(1); + expect(r.stderr).toContain('(INVENTORY_SOURCE_MISSING)'); + }); + + test('the real ledger passes --check', () => { + const r = run(['--check']); + expect(r.status, r.stderr).toBe(0); + }); +}); diff --git a/test/skill-bill.test.ts b/test/skill-bill.test.ts index c2a0d2924..3983cbc8c 100644 --- a/test/skill-bill.test.ts +++ b/test/skill-bill.test.ts @@ -47,7 +47,7 @@ describe('gstack-skill-bill', () => { const root = gstackRepo(); test('a gstack checkout skill reports every level and its last diet', () => { - const r = bill(root, ['demo']); + const r = bill(root, ['./demo']); expect(r.status).toBe(0); expect(r.stdout).toContain('SKILL: demo (gstack-checkout)'); expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/); @@ -58,7 +58,7 @@ describe('gstack-skill-bill', () => { test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => { fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200)); - const r = bill(root, ['demo', '--json']); + const r = bill(root, ['./demo', '--json']); const report = JSON.parse(r.stdout); expect(report.baseline.status).toBe('found'); expect(report.baseline.due).toBe(true); @@ -114,6 +114,37 @@ describe('gstack-skill-bill', () => { }); }); +describe('gstack-skill-bill EVIDENCE line (docs/inventory/evidence.json)', () => { + const root = path.join(tmp, 'evidence-repo'); + const ledger = (parts: Record) => { + fs.mkdirSync(path.join(root, 'docs', 'inventory'), { recursive: true }); + fs.writeFileSync(path.join(root, 'docs', 'inventory', 'evidence.json'), JSON.stringify({ schema_version: 1, parts })); + }; + fs.mkdirSync(path.join(root, 'bin'), { recursive: true }); + fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router'); + fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0'); + skill(path.join(root, 'demo'), 'demo', 'body\n', 'SKILL.md.tmpl'); + + test('each status prints its source, plan or reason; a skill with no row says so', () => { + ledger({ demo: { status: 'untested', grandfathered: true, plan: 'none' } }); + expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: none (grandfathered)'); + ledger({ demo: { status: 'untested', reason: 'r', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-1', measure_by: '2026-12-01' } }); + expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: test/fixtures/skill-evals/demo/tasks.json (owner GSTA-1, measure by 2026-12-01)'); + ledger({ demo: { status: 'helps', sources: ['run-9/results.json'], models: ['claude-opus-5-5'], prompt: 'preset', scope: 's', masked: true } }); + expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: helps sources: run-9/results.json'); + ledger({ demo: { status: 'safety', reason: 'blocks rm -rf' } }); + expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: safety reason: blocks rm -rf'); + ledger({}); + expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: missing: no demo row in docs/inventory/evidence.json'); + expect(JSON.parse(bill(root, ['./demo', '--json']).stdout).evidence).toMatchObject({ key: 'demo', status: 'missing' }); + }); + + test('the real checkout: a guard prints its safety reason', () => { + const r = spawnSync(BIN, ['careful'], { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf-8', timeout: 30_000 }); + expect(r.stdout).toMatch(/^EVIDENCE: safety reason: .+/m); + }); +}); + describe('gstack-skill-bill baselines and the root skill', () => { test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => { const root = path.join(tmp, 'gstack-root');