inventory: the R56 evidence ledger, gstack-inventory and skill-bill EVIDENCE

docs/inventory/evidence.json holds one row per inventory key: the skill
folders discoverTemplates() finds (root skill as gstack, symlinked folders
excluded) plus seven shared parts, 71 keys. lib/inventory.ts validates each
row by its status's own fields, with named codes (LEDGER_*), and joins the
ledger with lib/context-bill.ts buildBill. bin/gstack-inventory writes
docs/inventory.md, --check validates (exit 1 with codes), --json prints.
Paid dispatches per run for /ship and /autoplan are quoted from their own
docs and found by line, so a moved citation fails --check
(INVENTORY_SOURCE_MISSING). Every current row is grandfathered and seeded
from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json):
in-flight fair tests and proposed registry rows as plans, earlier unfair
results under prior with why (stub, unmasked, ceiling, ...), five safety
rows. gstack-skill-bill prints the skill's EVIDENCE line.

Evidence: plan none (owner ruling R56, GST-914); the ledger itself is the measurement record
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-11-02 05:03:32 +00:00
1 parent a26dfe9940
commit 37f515bae0
8 files changed
+1208 -4

No files matched your search

+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
// gstack-inventory — every gstack part, its load per run and its evidence (ETHOS.md: 9. Earn Its Place). Logic: lib/inventory.ts.
import { writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { buildInventory, checkInventory, DOC_PATH, LEDGER_PATH, renderInventory } from "../lib/inventory";
const USAGE = `Usage: gstack-inventory [--root <gstack checkout>] [--check | --json]
gstack-inventory --help
Joins the evidence ledger (${LEDGER_PATH}) with gstack-context-bill's sizes and writes ${DOC_PATH}:
parts ranked by load per run, each part's evidence status, a "measure next" order (largest untested
first), shared-part sizes, paid dispatches per run for /ship and /autoplan, and the key list.
--check validates the ledger (every key has one row; each row has its status's fields), the quoted
dispatch sources and the shared-part files, and writes nothing. --json prints the inventory.
Exit 0 when clean; 1 with one "<key>: <problem> (<CODE>)" line per problem; 2 on a bad argument.`;
const args = process.argv.slice(2);
if (args.includes("--help") || args.includes("-h")) { console.log(USAGE); process.exit(0); }
const rootAt = args.indexOf("--root");
const root = resolve(rootAt >= 0 ? args[rootAt + 1] ?? "" : join(import.meta.dir, ".."));
const unknown = args.find((a, i) => !["--check", "--json", "--root"].includes(a) && !(rootAt >= 0 && i === rootAt + 1));
if (unknown !== undefined || (rootAt >= 0 && !args[rootAt + 1]) || (args.includes("--check") && args.includes("--json"))) {
console.error(unknown !== undefined ? `gstack-inventory: unknown argument '${unknown}'; valid: --root <dir> --check --json --help` : USAGE);
process.exit(2);
}
const report = (problems: { key: string; code: string; message: string }[]) => {
for (const p of problems) console.error(`gstack-inventory: ${p.key}: ${p.message} (${p.code})`);
};
if (args.includes("--check")) {
const checked = checkInventory(root);
report(checked.problems);
if (checked.problems.length) process.exit(1);
console.log(`INVENTORY: ok keys=${checked.keys.length} ledger=${LEDGER_PATH} dispatch_sources=${checked.dispatches.length}`);
process.exit(0);
}
const inv = buildInventory(root);
report(inv.problems);
if (args.includes("--json")) { console.log(JSON.stringify(inv, null, 2)); process.exit(inv.problems.length ? 1 : 0); }
if (inv.problems.length) { console.error(`gstack-inventory: ${DOC_PATH} not written; fix the problems above, then rerun`); process.exit(1); }
writeFileSync(join(root, DOC_PATH), renderInventory(inv));
console.log(`INVENTORY: wrote ${DOC_PATH} keys=${inv.keys.length} commit=${inv.commit}`);
+2 -1
View File
@@ -11,7 +11,8 @@ Names the skill's kind: gstack-checkout (edit SKILL.md.tmpl, then bun run gen:sk
gstack-install (never edit; gstack-upgrade overwrites it) or user. BASELINE is the newest commit
with a "Skill-Diet: <name>" trailer; DUE means the authored file grew over 10% since then.
In a gstack checkout, EVAL TASKS names test/fixtures/skill-evals/<name>/tasks.json (or none) and
EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format.
EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format, and
EVIDENCE prints the skill's row in docs/inventory/evidence.json: its status and source, plan or reason.
A name must match exactly; pass a path when it names more than one skill.
Exit 0 on a report, 2 on SKILL_NOT_FOUND, SKILL_AMBIGUOUS or SKILL_MD_MISSING (each with a fix line).`;
+623
View File
@@ -0,0 +1,623 @@
{
"schema_version": 1,
"note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.",
"seed_source": "org-gstack-gbrain/research/gstack-part-evidence.json (GEVA-156 for GST-915, 2026-11-02, gstack-evals main 409ff47)",
"parts": {
"autoplan": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: autoplan: untested (never measured)"
},
"benchmark": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: benchmark: untested (never measured)"
},
"benchmark-models": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: benchmark-models: untested (never measured)"
},
"browse": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: browse: untested (never measured)"
},
"canary": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: canary: untested (never measured)"
},
"captain": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: captain: untested (never measured)"
},
"careful": {
"status": "safety",
"grandfathered": true,
"reason": "safety guard: warns before destructive commands (rm -rf, DROP TABLE, force-push); kept on that reason, an average score cannot see the rare bad run it stops",
"seed": "GEVA-2, 2026-11-02: careful: exempt (safety guard)"
},
"claude-code": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: claude-code: untested (never measured)"
},
"codex": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "not in GEVA-2's table (2026-11-02); never measured"
},
"context-restore": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: context-restore: untested (never measured)"
},
"context-save": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: context-save: untested (never measured)"
},
"cso": {
"status": "safety",
"grandfathered": true,
"reason": "security: supported security findings with explicit coverage; kept on its security reason (GEVA-2 marks it exempt; GStack Minder may reclassify it)",
"seed": "GEVA-2, 2026-11-02: cso: exempt (security audit skill, treated as exempt; GStack Minder may reclassify)"
},
"design-consultation": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: design-consultation: untested (never measured)"
},
"design-html": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: design-html: untested (never measured)"
},
"design-motion": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: design-motion: untested (never measured)"
},
"design-review": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-9 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: design-review: untested (never measured)"
},
"design-shotgun": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: design-shotgun: untested (never measured)"
},
"deslop-shared-libs": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: deslop-shared-libs: untested (never measured)"
},
"devex-review": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: devex-review: untested (never measured)"
},
"diagram": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: diagram: untested (never measured)"
},
"document-generate": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: document-generate: untested (never measured)"
},
"document-release": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: document-release: untested (never measured)"
},
"eval-plan": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: eval-plan: untested (never measured)"
},
"freeze": {
"status": "safety",
"grandfathered": true,
"reason": "safety guard: hard-blocks edits outside one directory; kept on that reason",
"seed": "GEVA-2, 2026-11-02: freeze: exempt (safety guard)"
},
"gstack": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: gstack (root skill): untested (never measured)"
},
"gstack-upgrade": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: upgrade: untested (never measured)"
},
"guard": {
"status": "safety",
"grandfathered": true,
"reason": "safety guard: careful and freeze together; kept on that reason",
"seed": "GEVA-2, 2026-11-02: guard: exempt (safety guard)"
},
"health": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: health: untested (never measured)"
},
"investigate": {
"status": "untested",
"grandfathered": true,
"plan": "R3 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-101-r48-follow-ups-prereg (#112, main f0b81c0) and its amendment (open PR #133), told /investigate vs no gstack on SWE-bench Verified, preset, Opus 5.5 then GPT-6.1 Sol",
"prior": [
{
"ref": "gstack-evals #109 stacks round 1 TB2 (GEVA-89, GEVA-93), wiki/board/2026-11-01-geva-93-stacks-round-1-tb2-results",
"why": [
"stub",
"no-exposure",
"unmasked"
],
"note": "Opus +3 points [-3, +9] at 1.9x cost; debug report in 0 of 111 trials; particle-1 -5 [-14, +1]"
},
{
"ref": "GSTA-12 Lane B quick look 2, wiki/board/2026-10-29-gsta-12-laneb-quick-2-results",
"why": [
"stub",
"unmasked"
],
"note": "Opus gstack minus none -7 [-13, -2] at 1.7x cost; did not replicate"
},
{
"ref": "GSTA-30 Lane B quick look 3 (#89), wiki/board/2026-10-30-gsta-30-laneb-quick-3-results",
"why": [
"stub",
"small-tier"
],
"note": "gstack arms on Sonnet, particle-1, DeepSeek and Haiku; Opus ran the none arm only"
},
{
"ref": "HARN-14 replication (harness-evals #181, unmerged)",
"why": [
"stub",
"small-tier"
],
"note": "Haiku 5.5: gstack minus none +3.5 [-12.3, +20.0]"
}
],
"r48": {
"verdict": "FIX-METHOD",
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R3 is the fix"
},
"seed": "GEVA-2, 2026-11-02: investigate: measurement under way (R3)"
},
"ios-clean": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ios-clean: untested (never measured)"
},
"ios-design-review": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ios-design-review: untested (never measured)"
},
"ios-fix": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ios-fix: untested (never measured)"
},
"ios-qa": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ios-qa: untested (never measured)"
},
"ios-sync": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ios-sync: untested (never measured)"
},
"land-and-deploy": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: land-and-deploy: untested (never measured)"
},
"landing-report": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: landing-report: untested (never measured)"
},
"learn": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"prior": [
{
"ref": "GSTA-13, GEVA-54 and GEVA-65 Lane A runs (wiki/board/2026-10-28-gsta-13-quick-look-3-results and the GEVA-54 and GEVA-65 posts)",
"why": [
"stub",
"no-true-no-gstack-arm",
"ceiling"
],
"note": "Opus contract tasks 27/30 vs 27/30"
},
{
"ref": "GEVA-65 Lane A rerun row 5 (GPT-6.1 Sol in Codex)",
"why": [
"not-claude-code",
"no-true-no-gstack-arm"
],
"note": "Sol failed the no-code-change safety task in 5 of 9 skill trials vs 0 of 3 without; small n, a lead to check, not a hurts verdict"
}
],
"seed": "GEVA-2, 2026-11-02: learn: untested (unfair test: SDK stub; no true no-gstack arm; Opus at ceiling; Sol ran in Codex, not Claude Code)"
},
"make-pdf": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: make-pdf: untested (never measured)"
},
"modernize": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: modernize: untested (never measured)"
},
"office-hours": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"prior": [
{
"ref": "GSTA-13 quick look 3 Lane A, wiki/board/2026-10-28-gsta-13-quick-look-3-results",
"why": [
"stub",
"no-true-no-gstack-arm"
],
"note": "contract tasks on Opus: gstack 74% vs none 28%; the tasks grade the skill's own contract, a manipulation check, not lift"
},
{
"ref": "GEVA-65 Lane A rerun, wiki/board/2026-10-31-geva-65-lanea-shakedown-rerun-results",
"why": [
"stub",
"no-true-no-gstack-arm"
],
"note": "Opus before 0.67, after 0.76, none 0.36"
}
],
"seed": "GEVA-2, 2026-11-02: office-hours: untested (unfair test: SDK stub; no true no-gstack arm; tasks grade the skill's own contract, a manipulation check, not lift)"
},
"open-gstack-browser": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: open-gstack-browser: untested (never measured)"
},
"pair-agent": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: pair-agent: untested (never measured)"
},
"plan-ceo-review": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-5 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"prior": [
{
"ref": "CEO panel v2 admission (GEVA-142), wiki/board/2026-11-02-geva-142-ceo-panel-v2-admission-results (open PR #131)",
"why": [
"ceiling",
"unmasked"
],
"note": "preset, Opus 5.5 only: must-fix recall 1.00 in all three arms; plain invocation ran the procedure in 0 of 16"
},
{
"ref": "Planning R1a pilot (GEVA-102), wiki/board/2026-11-01-geva-102-planning-r1a-pilot-stop (#114)",
"why": [
"stub",
"ceiling",
"unmasked"
],
"note": "none-opus 8/8; gstack-opus read the sections file in 0 of 4"
}
],
"r48": {
"verdict": "FIX-METHOD",
"ref": "CEO panel v2 admission (GEVA-142, open PR #131): FIX-METHOD on the panel, a harder panel is planned"
},
"seed": "GEVA-2, 2026-11-02: plan-ceo-review: untested (unfair test: ceiling, control 12 of 12 so no MDD; plain invocation ran the procedure in 0 of 16)"
},
"plan-design-review": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: plan-design-review: untested (never measured)"
},
"plan-devex-review": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: plan-devex-review: untested (never measured)"
},
"plan-eng-review": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: plan-eng-review: untested (never measured)"
},
"plan-tune": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: plan-tune: untested (never measured)"
},
"qa": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: qa: untested (never measured)"
},
"qa-only": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: qa-only: untested (never measured)"
},
"retro": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: retro: untested (never measured)"
},
"review": {
"status": "untested",
"grandfathered": true,
"plan": "R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms",
"prior": [
{
"ref": "gstack-evals #108 stacks round 1 review (GEVA-89), wiki/board/2026-11-01-geva-89-stacks-round-1-review-results",
"why": [
"stub",
"no-exposure",
"unmasked"
],
"note": "Opus 5.5 blocking recall 9.5% vs 7.1%, +2.4 points [-6.4, +11.1]; specialists and adversarial pass ran in 0 of 66 sessions"
},
{
"ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results",
"why": [
"stub",
"unmasked"
],
"note": "gstack-opus minus none-opus +3.5 points [-9.6, +16.7]"
},
{
"ref": "GSTA-11 quick look 1 review replay, wiki/board/2026-10-28-gsta-11-review-replay-results",
"why": [
"not-claude-code",
"unmasked"
],
"note": "/review's prompts through the Messages API; Opus recall 44% vs 39% plain, 12 bugs"
}
],
"r48": {
"verdict": "FIX-METHOD",
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R2 is the fix"
},
"seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)"
},
"scrape": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: scrape: untested (never measured)"
},
"setup-browser-cookies": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: setup-browser-cookies: untested (never measured)"
},
"setup-deploy": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: setup-deploy: untested (never measured)"
},
"setup-gbrain": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: setup-gbrain: untested (never measured)"
},
"ship": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"seed": "GEVA-2, 2026-11-02: ship: untested (never measured)"
},
"skillify": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: skillify: untested (never measured)"
},
"spec": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: spec: untested (never measured)"
},
"sync-gbrain": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: sync-gbrain: untested (never measured)"
},
"test-audit": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: test-audit: untested (never measured)"
},
"ubicloud": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: ubicloud: untested (never measured)"
},
"unfreeze": {
"status": "safety",
"grandfathered": true,
"reason": "safety guard: the off switch for freeze's edit restriction; kept with the guard it releases",
"seed": "GEVA-2, 2026-11-02: unfreeze: exempt (safety guard (undoes freeze))"
},
"verify": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "not in GEVA-2's table (2026-11-02); never measured"
},
"write-skill": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: write-skill: untested (never measured)"
},
"preamble": {
"status": "untested",
"grandfathered": true,
"plan": "I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1",
"prior": [
{
"ref": "GEVA-100 exposure counts, org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100)",
"why": [
"stub",
"no-exposure"
],
"note": "ceremony sections ran in 0 of 66 Opus review sessions and 6 of 111 Opus TB2 trials; no effect estimate"
}
],
"r48": {
"verdict": "CUT-CANDIDATE",
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); taken by Garry as the I1 ceremony-cut eval"
},
"seed": "GEVA-2, 2026-11-02: preamble (ceremony): measurement under way (I1, ceremony-cut eval)"
},
"catalog": {
"status": "untested",
"grandfathered": true,
"plan": "Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description",
"prior": [
{
"ref": "GSTA-12 Lane B quick look 1, wiki/board/2026-10-29-gsta-12-laneb-quick-results",
"why": [
"stub",
"no-exposure",
"unmasked"
],
"note": "untold skill opened in 0 of 180 gstack-arm trials; Opus gstack minus none +5 [-3, +17]"
}
],
"seed": "GEVA-2, 2026-11-02: always-on skill catalog (descriptions): measurement under way (Run D)"
},
"capy-skill": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: capy/skill/gstack/SKILL.md (Capy context): untested (never measured)",
"note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it"
},
"capy-factory": {
"status": "untested",
"grandfathered": true,
"plan": "none",
"seed": "GEVA-2, 2026-11-02: capy/skill/gstack/factory.md (Capy context): untested (never measured)",
"note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it"
},
"ship-review-panel": {
"status": "untested",
"grandfathered": true,
"plan": "GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
"prior": [
{
"ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results",
"why": [
"stub",
"not-claude-code",
"unmasked"
],
"note": "arm C minus none +56.9 points [40.5, 72.2]; the none arm ran on the stub, arm C in Capy at another gstack version, on a cohort mined from gstack's own catches"
},
{
"ref": "GSTA-48 review no-loss arm D (#77), wiki/board/2026-10-30-gsta-48-review-noloss-arm-how-to",
"why": [
"variant-only",
"not-claude-code"
],
"note": "arm D vs pooled arm C, no no-gstack arm"
}
],
"seed": "GEVA-2, 2026-11-02: ship: review panel (specialist, adversarial, targeted, outside voice, red team): untested (unfair test: none arm on the SDK stub; arms from different harnesses and gstack versions; cohort mined from gstack's own catches)"
},
"ship-docs-audit": {
"status": "untested",
"grandfathered": true,
"plan": "folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered",
"seed": "GEVA-2, 2026-11-02: ship: docs audit (Step 14.5): untested (never measured)"
},
"autoplan-phases": {
"status": "untested",
"grandfathered": true,
"plan": "folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered",
"seed": "GEVA-2, 2026-11-02: autoplan: CEO, design, DX and eng phases, outside voice: untested (never measured)"
}
},
"unmapped": [
{
"row": "review: Review Army and adversarial pass",
"why": "a section inside /review, not its own key; R2 measures it as part of the review row's plan"
},
{
"row": "ship: QA probes (Step 9.2.1)",
"why": "a step inside /ship with no separate size; folded into the ship row (GEVA-2 registry row R56-4)"
},
{
"row": "gstack as a whole (installed, any skill)",
"why": "not a single text part; GEVA-2's real-world outcome lane measures it (gstack-evals plan #111), not a measurement plan for any one part"
}
]
}
+313
View File
@@ -0,0 +1,313 @@
/**
* inventory — every part of gstack, its load per run and its evidence (ETHOS.md: 9. Earn Its Place).
*
* Keys are the skill directories discoverTemplates() finds (the root skill as `gstack`) plus
* SHARED_KEYS. The ledger, docs/inventory/evidence.json, is hand-kept: each row is checked by its
* status's own fields (validateEntry). buildInventory joins the ledger with lib/context-bill.ts
* buildBill (offline token estimates) and the dispatch counts ship and autoplan already document,
* each found by its quoted line so a citation that drifts is named. Read-only except the CLI's write.
*/
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { discoverTemplates } from '../scripts/discover-skills';
import { buildBill, contentClass, TOKEN_DIVISOR, TOKEN_DIVISORS } from './context-bill';
export const LEDGER_PATH = 'docs/inventory/evidence.json';
export const DOC_PATH = 'docs/inventory.md';
export const ROOT_KEY = 'gstack';
export interface SharedPart { key: string; label: string; load: string; files: string[] }
/** Shared parts: sized from their files (catalog from the bill's frontmatter). */
export const SHARED_PARTS: SharedPart[] = [
{ key: 'preamble', label: 'shared preamble (sections/preamble.md, identical in every tier 2+ skill)', load: 'every tier 2+ skill run, inside that skill\'s load', files: ['ship/sections/preamble.md'] },
{ key: 'catalog', label: 'skill catalog (every skill\'s frontmatter)', load: 'every session', files: [] },
{ key: 'capy-skill', label: 'capy/skill/gstack/SKILL.md', load: 'every Capy session', files: ['capy/skill/gstack/SKILL.md'] },
{ key: 'capy-factory', label: 'capy/skill/gstack/factory.md', load: 'every Capy session', files: ['capy/skill/gstack/factory.md'] },
{ key: 'ship-review-panel', label: '/ship review panel (review-army and adversarial sections)', load: 'every /ship run, by depth', files: ['ship/sections/review-army.md', 'ship/sections/adversarial.md'] },
{ key: 'ship-docs-audit', label: '/ship docs audit (Step 14.5)', load: 'every /ship run', files: ['ship/sections/documentation.md'] },
{ key: 'autoplan-phases', label: '/autoplan CEO, design, DX and eng phases', load: 'every /autoplan run, by scope', files: ['autoplan/sections/ceo-phase.md', 'autoplan/sections/design-phase.md', 'autoplan/sections/dx-phase.md', 'autoplan/sections/eng-phase.md'] },
];
export const SHARED_KEYS = SHARED_PARTS.map(p => p.key);
export interface Dispatch { skill: 'ship' | 'autoplan'; what: string; count: string; file: string; quote: string }
/** Paid dispatches per run, from the budget tables and docs that already state them (no new instrumentation). */
export const DISPATCHES: Dispatch[] = [
{ skill: 'ship', what: 'Step 9 review, pass 1, light depth, docs or tests only', count: '1 reader (the core checklist)', file: 'docs/review-policy.md', quote: '| **light**, runtime 0 |' },
{ skill: 'ship', what: 'Step 9 review, pass 1, light depth, code', count: '3 readers (core, one batched specialist, native adversarial), plus the outside voice on a diff that publishes numbers', file: 'docs/review-policy.md', quote: '| **light**, runtime above 0 |' },
{ skill: 'ship', what: 'Step 9 review, pass 1, standard depth', count: 'the batched specialist, the Red Team at 200 runtime lines, adversarial and outside voice', file: 'docs/review-policy.md', quote: '| **standard** |' },
{ skill: 'ship', what: 'Step 9 review, pass 1, full depth', count: 'the batched specialist on the whole diff and the Red Team always', file: 'docs/review-policy.md', quote: '| **full** |' },
{ skill: 'ship', what: 'targeted reader on money, gate, auth or secret code', count: '+1 at every depth', file: 'docs/review-policy.md', quote: '`gstack-review-plan` dispatches one' },
{ skill: 'ship', what: 'audits (Steps 7, 8 and 14.5)', count: '3', file: 'lib/ship-budget-calibration.json', quote: 'each audit dispatch (Steps 7, 8, 14.5)' },
{ skill: 'ship', what: 'measured: one /ship child (gstack-evals#31)', count: '10 review dispatches', file: 'lib/ship-budget-calibration.json', quote: 'its 10 review dispatches' },
{ skill: 'ship', what: 'prior cost of one dispatch on the Capy meter', count: '7,430,000 tokens', file: 'lib/ship-budget-calibration.json', quote: '"per_dispatch": 7430000' },
{ skill: 'autoplan', what: 'voices per phase (native and outside)', count: '2', file: 'autoplan/DRIVER.md.tmpl', quote: 'per voice. Dispatch the native prompt to a subagent on your model and the outside prompt to' },
{ skill: 'autoplan', what: 'phases per round: CEO and Eng always, Design with UI scope, DX with developer-facing scope', count: '2 to 4, so 4 to 8 voice dispatches a round', file: 'autoplan/SKILL.md.tmpl', quote: 'Phases MUST run in strict order: CEO → Design (if UI scope) → DX (if developer-facing' },
{ skill: 'autoplan', what: 'CEO spec review loop', count: 'up to 3 launches', file: 'autoplan/sections/ceo-phase.md.tmpl', quote: 'three-launch cap' },
{ skill: 'autoplan', what: 'review rounds per phase (new unattended runs)', count: 'up to 2', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_MAX_ROUNDS = 2;' },
{ skill: 'autoplan', what: 'reserved per voice on Capy or under a spend cap', count: '$5', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_HOST_ESTIMATE_USD = 5;' },
];
export const STATUSES = ['helps', 'no-effect', 'hurts', 'untested', 'safety'] as const;
export type Status = (typeof STATUSES)[number];
export const R48_VERDICTS = ['KEEP', 'FIX-METHOD', 'CUT-CANDIDATE'] as const;
/** R54: the frontier models under test; a row may add at most one named low-cost primary, never Haiku or Sonnet. */
export const FRONTIER_MODELS = ['claude-opus-5-5', 'gpt-6.1-sol'];
/** Why an earlier result does not count as fair evidence yet. */
export const PRIOR_WHY: Record<string, string> = {
stub: 'ran on the Agent SDK stub prompt, not the claude_code preset',
'small-tier': 'Haiku, Sonnet or another non-frontier model only',
unmasked: 'no mask recorded: the no-gstack arm predates the masking rule and was not re-checked',
ceiling: 'control at the ceiling, so no difference could show',
'no-true-no-gstack-arm': 'the none arm still had gstack installed',
'no-exposure': 'the model never used the part',
'not-claude-code': 'ran outside Claude Code (Messages API, Codex or Capy)',
'variant-only': 'variant against variant, no no-gstack arm',
};
const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'seed', 'note'];
export type LedgerCode =
| 'LEDGER_PARSE' | 'LEDGER_KEY_MISSING' | 'LEDGER_KEY_EXTRA' | 'LEDGER_STATUS_INVALID' | 'LEDGER_FIELD_UNKNOWN'
| 'LEDGER_SOURCES_MISSING' | 'LEDGER_MODELS_NOT_FRONTIER' | 'LEDGER_PROMPT_NOT_PRESET' | 'LEDGER_SCOPE_MISSING'
| 'LEDGER_UNMASKED' | 'LEDGER_R48_MISSING' | 'LEDGER_PLAN_MISSING' | 'LEDGER_PLAN_NONE_NEW' | 'LEDGER_PLAN_NOT_CONCRETE'
| 'LEDGER_REASON_MISSING' | 'LEDGER_OWNER_MISSING' | 'LEDGER_MEASURE_BY_MISSING' | 'LEDGER_PRIOR_INVALID'
| 'INVENTORY_SOURCE_MISSING' | 'INVENTORY_PART_MISSING';
export interface LedgerProblem { key: string; code: LedgerCode; message: string }
export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; seed?: string; note?: string }
export interface Ledger { parts: Record<string, LedgerEntry>; unmapped?: Array<{ row: string; why: string }>; [k: string]: unknown }
const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v);
const str = (v: unknown): v is string => typeof v === 'string' && v.trim() !== '';
const validR48 = (v: unknown) => isObj(v) && (R48_VERDICTS as readonly string[]).includes(v.verdict as string) && str(v.ref);
function frontierModels(v: unknown): boolean {
if (!Array.isArray(v) || !v.length || !v.every(str)) return false;
const ids = v.map(m => m.slice(m.lastIndexOf('/') + 1));
const other = ids.filter(m => !FRONTIER_MODELS.includes(m));
return ids.length > other.length && other.length <= 1 && !other.some(m => /haiku|sonnet/i.test(m));
}
/** Problems with one ledger row, by its status's own fields (helps, no-effect, hurts, untested, safety). */
export function validateEntry(key: string, entry: unknown): LedgerProblem[] {
const problems: LedgerProblem[] = [];
const add = (code: LedgerCode, message: string) => problems.push({ key, code, message });
if (!isObj(entry)) { add('LEDGER_STATUS_INVALID', 'the row is not an object'); return problems; }
for (const f of Object.keys(entry)) if (!FIELDS.includes(f)) add('LEDGER_FIELD_UNKNOWN', `unknown field ${f}; fields: ${FIELDS.join(', ')}`);
const prior = entry.prior;
if (prior !== undefined && !(Array.isArray(prior) && prior.every(p => isObj(p) && str(p.ref) && Array.isArray(p.why) && p.why.length > 0 && p.why.every(w => typeof w === 'string' && w in PRIOR_WHY))))
add('LEDGER_PRIOR_INVALID', `prior needs [{ref, why: [...]}] with why from: ${Object.keys(PRIOR_WHY).join(', ')}`);
const status = entry.status;
if (status === 'helps' || status === 'no-effect' || status === 'hurts') {
if (!Array.isArray(entry.sources) || !entry.sources.length || !entry.sources.every(str)) add('LEDGER_SOURCES_MISSING', `${status} needs sources: one or more run ids or results files`);
if (!frontierModels(entry.models)) add('LEDGER_MODELS_NOT_FRONTIER', `models must include ${FRONTIER_MODELS.join(' or ')} plus at most one named low-cost primary, never Haiku or Sonnet (R54)`);
if (entry.prompt !== 'preset') add('LEDGER_PROMPT_NOT_PRESET', 'prompt must be "preset" (the claude_code system prompt, not the Agent SDK stub)');
if (!str(entry.scope)) add('LEDGER_SCOPE_MISSING', 'scope must say what was measured');
if (entry.masked !== true) add('LEDGER_UNMASKED', 'masked must be true: the no-gstack arm was masked from the Capy drives and gstack installs');
if (status !== 'helps' && !validR48(entry.r48)) add('LEDGER_R48_MISSING', `${status} needs r48: {verdict: ${R48_VERDICTS.join('|')}, ref}`);
} else if (status === 'untested') {
const grandfathered = entry.grandfathered === true;
if (!str(entry.plan)) add('LEDGER_PLAN_MISSING', 'untested needs plan: a reference, or "none" on a grandfathered row');
else if (entry.plan === 'none') { if (!grandfathered) add('LEDGER_PLAN_NONE_NEW', 'plan: none is only for grandfathered rows; a new part names its measurement plan'); }
else if (!grandfathered && !/tasks\.json$|prereg/i.test(entry.plan)) add('LEDGER_PLAN_NOT_CONCRETE', 'a new part\'s plan is a test/fixtures/skill-evals/<name>/tasks.json or an /eval-plan preregistration');
if (!grandfathered) {
if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'a new untested part states its reason');
if (!str(entry.owner)) add('LEDGER_OWNER_MISSING', 'a new untested part names its owner');
if (!str(entry.measure_by) || !/^\d{4}-\d{2}-\d{2}$/.test(entry.measure_by) || Number.isNaN(Date.parse(entry.measure_by))) add('LEDGER_MEASURE_BY_MISSING', 'a new untested part needs measure_by: YYYY-MM-DD');
}
} else if (status === 'safety') {
if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'safety needs reason: what the guard protects');
} else add('LEDGER_STATUS_INVALID', `status must be one of ${STATUSES.join(', ')}`);
if (entry.r48 !== undefined && !validR48(entry.r48) && (status === 'untested' || status === 'safety')) add('LEDGER_R48_MISSING', `r48 needs {verdict: ${R48_VERDICTS.join('|')}, ref}`);
return problems;
}
/** Coverage (no missing, no extra key) then every row's own schema. */
export function validateLedger(ledger: unknown, keys: string[]): LedgerProblem[] {
if (!isObj(ledger) || !isObj(ledger.parts)) return [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `${LEDGER_PATH} needs a parts object` }];
const parts = ledger.parts;
return [
...keys.filter(k => !(k in parts)).map(k => ({ key: k, code: 'LEDGER_KEY_MISSING' as const, message: `no row in ${LEDGER_PATH}` })),
...Object.keys(parts).filter(k => !keys.includes(k)).map(k => ({ key: k, code: 'LEDGER_KEY_EXTRA' as const, message: 'not an inventory key (the key list is in docs/inventory.md)' })),
...keys.filter(k => k in parts).flatMap(k => validateEntry(k, parts[k])),
];
}
export function loadLedger(root: string): { ledger: Ledger | null; problems: LedgerProblem[] } {
try {
return { ledger: JSON.parse(fs.readFileSync(path.join(root, LEDGER_PATH), 'utf8')), problems: [] };
} catch (e) {
return { ledger: null, problems: [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `cannot read ${LEDGER_PATH}: ${(e as Error).message}` }] };
}
}
/** The skill keys (discoverTemplates, root skill as gstack, symlinked folders excluded) then SHARED_KEYS. */
export function inventoryKeys(root: string): string[] {
const skills = discoverTemplates(root).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d));
return [...skills.sort(), ...SHARED_KEYS];
}
/** Installed skill name to key: gstack-<name> is <name>; gstack-upgrade keeps its name. */
export const keyOf = (name: string) => (name !== 'gstack-upgrade' && name.startsWith('gstack-') ? name.slice('gstack-'.length) : name);
/** The status's evidence in one phrase: sources, plan or reason. */
export function evidenceRef(e: LedgerEntry): string {
if (e.status === 'untested') return `plan: ${e.plan}${e.grandfathered ? ' (grandfathered)' : ` (owner ${e.owner}, measure by ${e.measure_by})`}`;
if (e.status === 'safety') return `reason: ${e.reason}`;
return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}`;
}
/** gstack-skill-bill's EVIDENCE line for one skill folder in a gstack checkout. */
export function skillEvidence(root: string, dir: string): { key: string; status: Status | 'missing'; ref: string; line: string } {
const key = path.relative(root, dir) || ROOT_KEY;
const entry = loadLedger(root).ledger?.parts?.[key];
if (!entry) {
const ref = `no ${key} row in ${LEDGER_PATH}; add one (ETHOS.md: 9. Earn Its Place)`;
return { key, status: 'missing', ref, line: `EVIDENCE: missing: ${ref}` };
}
const ref = evidenceRef(entry);
return { key, status: entry.status, ref, line: `EVIDENCE: ${entry.status} ${ref}` };
}
export interface CitedDispatch extends Dispatch { line: number | null }
const tokensOf = (rel: string, bytes: number) => Math.round(bytes / (TOKEN_DIVISORS[contentClass(rel)] ?? TOKEN_DIVISOR));
/** Ledger, dispatch citations and shared-part files; no bill, so --check stays fast. */
export function checkInventory(root: string): { keys: string[]; ledger: Ledger | null; dispatches: CitedDispatch[]; problems: LedgerProblem[] } {
const keys = inventoryKeys(root);
const loaded = loadLedger(root);
const problems = [...loaded.problems, ...(loaded.ledger ? validateLedger(loaded.ledger, keys) : [])];
const dispatches = DISPATCHES.map(d => {
const file = path.join(root, d.file);
const idx = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').findIndex(l => l.includes(d.quote)) : -1;
if (idx < 0) problems.push({ key: `dispatch:${d.skill}`, code: 'INVENTORY_SOURCE_MISSING', message: `${d.file} no longer holds "${d.quote}"; update DISPATCHES in lib/inventory.ts` });
return { ...d, line: idx < 0 ? null : idx + 1 };
});
for (const part of SHARED_PARTS) for (const f of part.files) if (!fs.existsSync(path.join(root, f)))
problems.push({ key: part.key, code: 'INVENTORY_PART_MISSING', message: `${f} is missing; update SHARED_PARTS in lib/inventory.ts` });
return { keys, ledger: loaded.ledger, dispatches, problems };
}
export interface PartRow { key: string; kind: 'skill' | 'shared'; label: string; loads: string; loadTokens: number | null; alwaysOnTokens: number | null; status: Status | 'missing'; ref: string }
export interface Inventory {
commit: string; date: string; tokenSource: string; keys: string[]; parts: PartRow[];
tiers: Array<{ tier: string; skills: string[] }>; dispatches: CitedDispatch[];
prior: Array<{ key: string; ref: string; why: string[]; note?: string }>;
unmapped: Array<{ row: string; why: string }>; problems: LedgerProblem[];
}
function git(root: string, args: string[]): string | null {
const r = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8', timeout: 10_000 });
return r.status === 0 ? r.stdout.trim() : null;
}
/** The commit the snapshot measured, its date, and whether the tree had other changes. */
function measuredAt(root: string): { commit: string; date: string } {
const top = git(root, ['rev-parse', '--show-toplevel']);
if (!top || path.resolve(top) !== path.resolve(root)) return { commit: 'none', date: 'none' };
const dirty = git(root, ['status', '--porcelain', '--', '.', `:!${DOC_PATH}`]);
return { commit: (git(root, ['rev-parse', '--short=9', 'HEAD']) ?? 'none') + (dirty ? ' (with uncommitted changes)' : ''), date: git(root, ['log', '-1', '--format=%cs']) ?? 'none' };
}
/** Joins the ledger with the context bill: one row per key, ranked by load per run. */
export function buildInventory(root: string): Inventory {
const checked = checkInventory(root);
const bill = buildBill(root);
const resolved = path.resolve(root);
const bySkill = new Map(bill.skills.map(s => [path.relative(resolved, s.dir) || ROOT_KEY, s]));
const parts = checked.ledger?.parts ?? {};
const skillKeys = checked.keys.filter(k => !SHARED_KEYS.includes(k));
const evidence = (key: string) => (parts[key] ? { status: parts[key]!.status, ref: evidenceRef(parts[key]!) } : { status: 'missing' as const, ref: 'no row' });
const rows: PartRow[] = skillKeys.map(key => {
const s = bySkill.get(key);
return { key, kind: 'skill', label: key === ROOT_KEY ? 'root skill (/gstack router)' : `/${key}`, loads: 'every use', loadTokens: s ? Math.round(s.perInvocationTokens) : null, alwaysOnTokens: s ? Math.round(s.frontmatterTokens) : null, ...evidence(key) };
});
const catalog = skillKeys.reduce((n, k) => n + (bySkill.get(k)?.frontmatterTokens ?? 0), 0);
for (const part of SHARED_PARTS) {
const sizes = part.files.map(f => { const file = path.join(root, f); return fs.existsSync(file) ? tokensOf(f, fs.statSync(file).size) : 0; });
const tokens = part.key === 'catalog' ? Math.round(catalog) : sizes.reduce((a, b) => a + b, 0);
rows.push({ key: part.key, kind: 'shared', label: part.label, loads: part.load, loadTokens: tokens, alwaysOnTokens: part.key === 'catalog' ? tokens : null, ...evidence(part.key) });
}
rows.sort((a, b) => (b.loadTokens ?? -1) - (a.loadTokens ?? -1) || a.key.localeCompare(b.key));
const tiers = new Map<string, string[]>();
for (const key of skillKeys) {
const tmpl = path.join(root, key === ROOT_KEY ? '' : key, 'SKILL.md.tmpl');
const tier = /^preamble-tier:\s*(\d+)/m.exec(fs.readFileSync(tmpl, 'utf8'))?.[1] ?? 'none';
tiers.set(tier, [...(tiers.get(tier) ?? []), key]);
}
return {
...measuredAt(root),
tokenSource: bill.tokenSource,
keys: checked.keys,
parts: rows,
tiers: [...tiers].sort(([a], [b]) => a.localeCompare(b)).map(([tier, skills]) => ({ tier, skills })),
dispatches: checked.dispatches,
prior: checked.keys.flatMap(key => (parts[key]?.prior ?? []).map(p => ({ key, ...p }))),
unmapped: checked.ledger?.unmapped ?? [],
problems: checked.problems,
};
}
const n = (v: number | null) => (v === null ? 'n/a' : v.toLocaleString('en-US'));
const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/\n/g, ' ');
function renderRanking(inv: Inventory): string[] {
const out = ['## Parts by load per run', '', 'Load per run: a skill\'s SKILL.md plus the files it must read on every use (`perInvocationTokens` in lib/context-bill.ts); a shared part\'s files. Always-on: frontmatter every session\'s skill scanner reads.', '',
'| Rank | Part | Kind | Load per run (tokens) | Always-on (tokens) | Loads | Status | Source, plan or reason |', '|---|---|---|---|---|---|---|---|'];
inv.parts.forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${p.kind} | ${n(p.loadTokens)} | ${n(p.alwaysOnTokens)} | ${cell(p.loads)} | ${p.status} | ${cell(p.ref)} |`));
return out;
}
function renderMeasureNext(inv: Inventory): string[] {
const out = ['## Measure next', '', 'Untested parts, largest load per run first. A fair test runs the claude_code preset, frontier models (Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary) and a masked no-gstack arm ([TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm)). Safety rows are not on this list.', '',
'| Order | Part | Load per run (tokens) | Plan |', '|---|---|---|---|'];
inv.parts.filter(p => p.status === 'untested').forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${n(p.loadTokens)} | ${cell(p.ref.replace(/^plan: /, ''))} |`));
return out;
}
function renderShared(inv: Inventory): string[] {
const out = ['## Shared parts and preamble tiers', '', '| Tier | Skills | What loads |', '|---|---|---|'];
const what: Record<string, string> = { '1': 'a short preamble inline in SKILL.md', none: 'no preamble' };
for (const t of inv.tiers) out.push(`| ${t.tier} | ${t.skills.join(', ')} | ${what[t.tier] ?? 'the shared sections/preamble.md (row preamble), once per session'} |`);
out.push('', ...SHARED_PARTS.map(p => `- **${p.key}**: ${p.label}${p.files.length ? ` (${p.files.join(', ')})` : ''}; loads ${p.load}.`));
return out;
}
function renderDispatches(inv: Inventory): string[] {
const out = ['## Paid dispatches per run', '', 'Each count is quoted from the file and line named; `bin/gstack-inventory --check` fails when a quoted line moves away.', '', '| Skill | What | Count | Source |', '|---|---|---|---|'];
for (const d of inv.dispatches) out.push(`| ${d.skill} | ${cell(d.what)} | ${cell(d.count)} | ${d.line ? `${d.file}:${d.line}` : `${d.file}: source not found`} |`);
return out;
}
function renderPrior(inv: Inventory): string[] {
const out = ['## Earlier results that do not count yet', '', ...Object.entries(PRIOR_WHY).map(([k, v]) => `- \`${k}\`: ${v}.`), '', '| Part | Result | Why it does not count | Note |', '|---|---|---|---|'];
for (const p of inv.prior) out.push(`| ${p.key} | ${cell(p.ref)} | ${p.why.join(', ')} | ${cell(p.note ?? '')} |`);
return out;
}
function renderKeys(inv: Inventory): string[] {
const skills = inv.keys.filter(k => !SHARED_KEYS.includes(k));
return ['## Key list and aliases', '', `${inv.keys.length} keys: ${skills.length} skills (the folders \`discoverTemplates()\` finds, the root skill as \`gstack\`) and ${SHARED_KEYS.length} shared parts.`, '',
`- Skills: ${skills.join(', ')}.`, `- Shared: ${SHARED_KEYS.join(', ')}.`,
'- Installed names: `gstack-<name>` is key `<name>` (`gstack-ship` is `ship`); `gstack-upgrade` keeps its name; the root skill installs as `gstack`.'];
}
function renderNotHere(inv: Inventory): string[] {
return ['## Not in this inventory', '',
'- Out of scope: openclaw/skills/* and browser-skills/* (not gstack skills an agent loads from this checkout), and connect-chrome (a symlink to open-gstack-browser, counted once).',
...inv.unmapped.map(u => `- ${u.row}: ${u.why}`)];
}
export function renderInventory(inv: Inventory): string {
const count = (s: string) => inv.parts.filter(p => p.status === s).length;
const planned = inv.parts.filter(p => p.status === 'untested' && !p.ref.startsWith('plan: none')).length;
return [
'# gstack inventory: every part, its load per run and its evidence', '',
`Generated by \`bin/gstack-inventory\` from [evidence.json](inventory/evidence.json), the hand-kept ledger, and lib/context-bill.ts. Measured at commit ${inv.commit} (${inv.date}). This is a dated snapshot: rerun \`bin/gstack-inventory\` to refresh it; \`bin/gstack-inventory --check\` validates the ledger. The rule: [ETHOS.md: 9. Earn Its Place](../ETHOS.md#9-earn-its-place).`, '',
`Token counts are offline estimates (${inv.tokenSource}, about ±40%), not billed tokens.`, '',
'| Status | Parts |', '|---|---|',
...(['helps', 'no-effect', 'hurts'] as const).map(s => `| ${s} | ${count(s)} |`),
`| untested, with a plan | ${planned} |`, `| untested, plan none | ${count('untested') - planned} |`, `| safety | ${count('safety')} |`, '',
...renderRanking(inv), '', ...renderMeasureNext(inv), '', ...renderShared(inv), '', ...renderDispatches(inv), '',
...renderPrior(inv), '', ...renderKeys(inv), '', ...renderNotHere(inv), '',
].join('\n');
}
+7 -1
View File
@@ -11,6 +11,7 @@ import { homedir } from "node:os";
import { basename, dirname, join, relative, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import { estimateTokens } from "./context-bill";
import { skillEvidence } from "./inventory";
export const DUE_PCT = 10;
@@ -28,6 +29,8 @@ export interface SkillBillReport {
rows: LevelRow[];
baseline: Baseline;
evalTasks?: EvalTasks | null;
/** gstack checkout only: the skill's row in docs/inventory/evidence.json (ETHOS.md: 9. Earn Its Place). */
evidence?: ReturnType<typeof skillEvidence>;
warnings: string[];
}
@@ -196,8 +199,10 @@ export function buildSkillBill(dir: string): SkillBillReport {
for (const f of onDemandFiles(dir)) rows.push(row("on-demand", dir, f));
const warnings: string[] = [];
let evalTasks: EvalTasks | null | undefined;
let evidence: SkillBillReport["evidence"];
if (kind === "gstack-checkout") {
const root = gstackRootOf(dir)!;
evidence = skillEvidence(root, dir);
const file = join(root, "test", "fixtures", "skill-evals", name, "tasks.json");
evalTasks = existsSync(file) ? { ...checkEvalTasks(file), path: `test/fixtures/skill-evals/${name}/tasks.json` } : null;
if (evalTasks?.problems.length) warnings.push(`EVAL_TASKS_INVALID: ${evalTasks.problems[0]}${evalTasks.problems.length > 1 ? ` (+${evalTasks.problems.length - 1} more)` : ""}. fix: the task format in docs/designs/SKILL_EVAL_CONTRACT.md`);
@@ -206,7 +211,7 @@ export function buildSkillBill(dir: string): SkillBillReport {
warnings.push("RENDER_STALE: SKILL.md.tmpl changed but SKILL.md did not. fix: bun run gen:skill-docs");
}
if (kind === "gstack-install") warnings.push("INSTALLED_GSTACK_SKILL: gstack-upgrade overwrites this folder. fix: edit <skill>/SKILL.md.tmpl in a gstack checkout, or copy the skill to a folder you own");
return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, warnings };
return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, evidence, warnings };
}
export function renderSkillBill(r: SkillBillReport): string {
@@ -217,6 +222,7 @@ export function renderSkillBill(r: SkillBillReport): string {
else lines.push(`BASELINE: ${b.status} (${b.reason})`);
if (r.evalTasks) lines.push(`EVAL TASKS: ${r.evalTasks.path} (${r.evalTasks.count} tasks)`);
else if (r.evalTasks === null) lines.push(`EVAL TASKS: none; a diet writes test/fixtures/skill-evals/${r.name}/tasks.json before any cut`);
if (r.evidence) lines.push(r.evidence.line);
lines.push(...r.warnings);
lines.push(`Commit line: Skill-Size: <before>→<after> bytes (${r.authored})`);
return lines.join("\n");
+1
View File
@@ -37,6 +37,7 @@
{ "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" },
{ "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" },
{ "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" },
{ "file": "lib/inventory.ts", "movedFrom": "(new: the R56 evidence ledger check and the inventory behind bin/gstack-inventory, GST-914)" },
{ "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" },
{ "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" },
{ "file": "lib/owner-brief.ts", "movedFrom": "(new: the one-page owner brief and draft-direction close behind bin/gstack-owner-brief, plan E3)" },
+188
View File
@@ -0,0 +1,188 @@
/**
* R56 (ETHOS.md: 9. Earn Its Place): every part of gstack has a row in
* docs/inventory/evidence.json, validated by its status's own fields, and
* bin/gstack-inventory ranks the parts by load per run into docs/inventory.md.
*/
import { afterAll, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { discoverTemplates } from '../scripts/discover-skills';
import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory';
const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-inventory');
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-inventory-'));
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
const codes = (key: string, entry: unknown) => validateEntry(key, entry).map(p => p.code);
const run = (args: string[]) => spawnSync(BIN, args, { cwd: ROOT, encoding: 'utf8', timeout: 120_000 });
const VERDICT = { sources: ['gstack-evals run r9 results.json'], models: ['claude-opus-5-5', 'gpt-6.1-sol'], prompt: 'preset', scope: 'the full procedure, told', masked: true };
const NEW_UNTESTED = { status: 'untested', reason: 'catches missed migrations', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-168', measure_by: '2026-12-01' };
describe('the key list', () => {
test('keys are the skill directories discoverTemplates finds, the root skill as gstack, then the fixed shared keys', () => {
const keys = inventoryKeys(ROOT);
const skills = discoverTemplates(ROOT).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d));
expect(keys).toEqual([...skills.sort(), ...SHARED_KEYS]);
expect(keys).toContain('gstack');
expect(keys).toContain('gstack-upgrade');
expect(keys).not.toContain('connect-chrome');
expect(new Set(keys).size).toBe(keys.length);
});
test('installed names map to keys: gstack-<name> to <name>, gstack-upgrade stays literal', () => {
expect(keyOf('gstack-ship')).toBe('ship');
expect(keyOf('ship')).toBe('ship');
expect(keyOf('gstack-upgrade')).toBe('gstack-upgrade');
expect(keyOf('gstack')).toBe('gstack');
});
});
describe('the ledger covers every key and validates', () => {
const ledger = JSON.parse(fs.readFileSync(path.join(ROOT, LEDGER_PATH), 'utf8'));
test('no missing and no extra keys, and every row passes its status schema', () => {
const keys = inventoryKeys(ROOT);
expect(Object.keys(ledger.parts).filter(k => !keys.includes(k))).toEqual([]);
expect(keys.filter(k => !(k in ledger.parts))).toEqual([]);
expect(validateLedger(ledger, keys)).toEqual([]);
});
test('the initial backlog is grandfathered and the guards are safety rows', () => {
for (const [key, row] of Object.entries<any>(ledger.parts)) expect([key, row.grandfathered]).toEqual([key, true]);
for (const key of ['careful', 'freeze', 'guard', 'unfreeze', 'cso']) expect(ledger.parts[key].status).toBe('safety');
for (const key of ['capy-skill', 'capy-factory']) expect(ledger.parts[key]).toMatchObject({ status: 'untested', plan: 'none' });
});
test('a missing and an extra key are each named', () => {
const keys = inventoryKeys(ROOT);
const { ship, ...rest } = ledger.parts;
const problems = validateLedger({ ...ledger, parts: { ...rest, 'no-such-part': ship } }, keys);
expect(problems.map(p => `${p.key} ${p.code}`)).toEqual(['ship LEDGER_KEY_MISSING', 'no-such-part LEDGER_KEY_EXTRA']);
});
});
describe('the per-status schema', () => {
test('passing rows: a grandfathered untested part with plan none, a new part with a dated plan, a fair verdict, a safety guard', () => {
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none' })).toEqual([]);
expect(codes('a', NEW_UNTESTED)).toEqual([]);
expect(codes('a', { ...NEW_UNTESTED, plan: 'docs/evals/demo/PREREG.md' })).toEqual([]);
expect(codes('a', { status: 'helps', ...VERDICT })).toEqual([]);
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1'] })).toEqual([]);
expect(codes('a', { status: 'no-effect', ...VERDICT, r48: { verdict: 'CUT-CANDIDATE', ref: 'why-null.md' } })).toEqual([]);
expect(codes('a', { status: 'safety', reason: 'blocks rm -rf' })).toEqual([]);
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['stub', 'unmasked'] }] })).toEqual([]);
});
test('a new untested part needs reason, a concrete plan, owner and measure_by; plan none is refused', () => {
const { measure_by, ...noDate } = NEW_UNTESTED;
expect(codes('a', noDate)).toEqual(['LEDGER_MEASURE_BY_MISSING']);
expect(codes('a', { ...NEW_UNTESTED, measure_by: 'soon' })).toEqual(['LEDGER_MEASURE_BY_MISSING']);
const { owner, ...noOwner } = NEW_UNTESTED;
expect(codes('a', noOwner)).toEqual(['LEDGER_OWNER_MISSING']);
const { reason, ...noReason } = NEW_UNTESTED;
expect(codes('a', noReason)).toEqual(['LEDGER_REASON_MISSING']);
expect(codes('a', { ...NEW_UNTESTED, plan: 'none' })).toEqual(['LEDGER_PLAN_NONE_NEW']);
expect(codes('a', { ...NEW_UNTESTED, plan: 'we will test it later' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']);
expect(codes('a', { status: 'untested', grandfathered: true })).toEqual(['LEDGER_PLAN_MISSING']);
});
test('helps, no-effect and hurts need sources, frontier models, the preset prompt, a scope and a masked no-gstack arm', () => {
const { sources, ...noSources } = VERDICT;
expect(codes('a', { status: 'helps', ...noSources })).toEqual(['LEDGER_SOURCES_MISSING']);
expect(codes('a', { status: 'helps', ...VERDICT, sources: [] })).toEqual(['LEDGER_SOURCES_MISSING']);
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-haiku-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'claude-sonnet-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1', 'deepseek-v4.1-flash'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
expect(codes('a', { status: 'helps', ...VERDICT, prompt: 'sdk-stub' })).toEqual(['LEDGER_PROMPT_NOT_PRESET']);
expect(codes('a', { status: 'hurts', ...VERDICT, scope: '', r48: { verdict: 'KEEP', ref: 'x' } })).toEqual(['LEDGER_SCOPE_MISSING']);
expect(codes('a', { status: 'helps', ...VERDICT, masked: false })).toEqual(['LEDGER_UNMASKED']);
});
test('no-effect and hurts without an R48 verdict and its reference are refused', () => {
expect(codes('a', { status: 'no-effect', ...VERDICT })).toEqual(['LEDGER_R48_MISSING']);
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'DROP', ref: 'x' } })).toEqual(['LEDGER_R48_MISSING']);
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'KEEP' } })).toEqual(['LEDGER_R48_MISSING']);
});
test('safety needs a reason; unknown statuses, unknown fields and bad prior entries are refused', () => {
expect(codes('a', { status: 'safety' })).toEqual(['LEDGER_REASON_MISSING']);
expect(codes('a', { status: 'exempt', reason: 'x' })).toEqual(['LEDGER_STATUS_INVALID']);
expect(codes('a', { status: 'safety', reason: 'x', source: 'typo' })).toEqual(['LEDGER_FIELD_UNKNOWN']);
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['vibes'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ why: ['stub'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
});
});
/** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */
function fixtureTree(dir: string, parts: (keys: string[]) => Record<string, unknown>): string {
const write = (rel: string, text: string) => { fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); fs.writeFileSync(path.join(dir, rel), text); };
const skill = (rel: string, name: string, tier: number, body: string) => {
const fm = `---\nname: ${name}\npreamble-tier: ${tier}\ndescription: |\n Does ${name} things.\n---\n`;
write(path.join(rel, 'SKILL.md.tmpl'), fm + body);
write(path.join(rel, 'SKILL.md'), fm + '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n' + body);
};
skill('', 'gstack', 1, 'router\n');
skill('alpha', 'alpha', 2, 'a'.repeat(4000) + '\n');
skill('beta', 'beta', 2, 'b'.repeat(1000) + '\n');
write('alpha/sections/preamble.md', 'p'.repeat(800));
write('beta/sections/preamble.md', 'p'.repeat(800));
for (const part of SHARED_PARTS) for (const f of part.files) write(f, 'x'.repeat(500) + '\n');
for (const d of DISPATCHES) if (!fs.existsSync(path.join(dir, d.file))) write(d.file, 'filler\n');
for (const d of DISPATCHES) fs.appendFileSync(path.join(dir, d.file), `${d.quote}\n`);
const keys = inventoryKeys(dir);
write(LEDGER_PATH, JSON.stringify({ schema_version: 1, parts: parts(keys), unmapped: [{ row: 'gstack as a whole', why: 'not a single text part' }] }, null, 2));
return dir;
}
const grandfathered = (keys: string[]) => Object.fromEntries(keys.map(k => [k, k === 'beta' ? { status: 'safety', grandfathered: true, reason: 'a guard' } : { status: 'untested', grandfathered: true, plan: 'none' }]));
describe('bin/gstack-inventory', () => {
test('determinism: two runs on a fixed tree write the same docs/inventory.md and the same --json', () => {
const dir = fixtureTree(path.join(tmp, 'fixed'), grandfathered);
const first = run(['--root', dir]);
expect(first.status, first.stderr).toBe(0);
const a = fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8');
const second = run(['--root', dir]);
expect(second.status, second.stderr).toBe(0);
expect(fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8')).toBe(a);
expect(run(['--root', dir, '--json']).stdout).toBe(run(['--root', dir, '--json']).stdout);
expect(a).toContain('Measured at commit none');
expect(a.indexOf('| alpha |')).toBeLessThan(a.indexOf('| beta |'));
const start = a.indexOf('## Measure next');
const next = a.slice(start, a.indexOf('\n## ', start + 1));
expect(next).toContain('alpha');
expect(next).not.toContain('| beta |');
expect(a).toContain('gstack as a whole');
expect(a).toContain('gstack-<name>');
for (const d of DISPATCHES) expect(a).toContain(`${d.file}:`);
});
test('--check: ok on a complete ledger; a missing key exits 1 with its code', () => {
const dir = fixtureTree(path.join(tmp, 'check'), grandfathered);
const ok = run(['--root', dir, '--check']);
expect(ok.status, ok.stderr).toBe(0);
expect(ok.stdout).toMatch(/^INVENTORY: ok keys=\d+/m);
const bad = fixtureTree(path.join(tmp, 'check-bad'), keys => { const p = grandfathered(keys); delete p.alpha; return p; });
const r = run(['--root', bad, '--check']);
expect(r.status).toBe(1);
expect(r.stderr).toContain('alpha: no row in docs/inventory/evidence.json (LEDGER_KEY_MISSING)');
expect(run(['--root', bad]).status).toBe(1);
});
test('--check: a cited dispatch source that lost its quote is named', () => {
const dir = fixtureTree(path.join(tmp, 'drift'), grandfathered);
fs.writeFileSync(path.join(dir, DISPATCHES[0]!.file), 'rewritten\n');
const r = run(['--root', dir, '--check']);
expect(r.status).toBe(1);
expect(r.stderr).toContain('(INVENTORY_SOURCE_MISSING)');
});
test('the real ledger passes --check', () => {
const r = run(['--check']);
expect(r.status, r.stderr).toBe(0);
});
});
+33 -2
View File
@@ -47,7 +47,7 @@ describe('gstack-skill-bill', () => {
const root = gstackRepo();
test('a gstack checkout skill reports every level and its last diet', () => {
const r = bill(root, ['demo']);
const r = bill(root, ['./demo']);
expect(r.status).toBe(0);
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
@@ -58,7 +58,7 @@ describe('gstack-skill-bill', () => {
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
const r = bill(root, ['demo', '--json']);
const r = bill(root, ['./demo', '--json']);
const report = JSON.parse(r.stdout);
expect(report.baseline.status).toBe('found');
expect(report.baseline.due).toBe(true);
@@ -114,6 +114,37 @@ describe('gstack-skill-bill', () => {
});
});
describe('gstack-skill-bill EVIDENCE line (docs/inventory/evidence.json)', () => {
const root = path.join(tmp, 'evidence-repo');
const ledger = (parts: Record<string, unknown>) => {
fs.mkdirSync(path.join(root, 'docs', 'inventory'), { recursive: true });
fs.writeFileSync(path.join(root, 'docs', 'inventory', 'evidence.json'), JSON.stringify({ schema_version: 1, parts }));
};
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
skill(path.join(root, 'demo'), 'demo', 'body\n', 'SKILL.md.tmpl');
test('each status prints its source, plan or reason; a skill with no row says so', () => {
ledger({ demo: { status: 'untested', grandfathered: true, plan: 'none' } });
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: none (grandfathered)');
ledger({ demo: { status: 'untested', reason: 'r', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-1', measure_by: '2026-12-01' } });
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: test/fixtures/skill-evals/demo/tasks.json (owner GSTA-1, measure by 2026-12-01)');
ledger({ demo: { status: 'helps', sources: ['run-9/results.json'], models: ['claude-opus-5-5'], prompt: 'preset', scope: 's', masked: true } });
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: helps sources: run-9/results.json');
ledger({ demo: { status: 'safety', reason: 'blocks rm -rf' } });
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: safety reason: blocks rm -rf');
ledger({});
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: missing: no demo row in docs/inventory/evidence.json');
expect(JSON.parse(bill(root, ['./demo', '--json']).stdout).evidence).toMatchObject({ key: 'demo', status: 'missing' });
});
test('the real checkout: a guard prints its safety reason', () => {
const r = spawnSync(BIN, ['careful'], { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf-8', timeout: 30_000 });
expect(r.stdout).toMatch(/^EVIDENCE: safety reason: .+/m);
});
});
describe('gstack-skill-bill baselines and the root skill', () => {
test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => {
const root = path.join(tmp, 'gstack-root');