mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 03:00:26 +01:00
inventory: the R56 evidence ledger, gstack-inventory and skill-bill EVIDENCE
docs/inventory/evidence.json holds one row per inventory key: the skill folders discoverTemplates() finds (root skill as gstack, symlinked folders excluded) plus seven shared parts, 71 keys. lib/inventory.ts validates each row by its status's own fields, with named codes (LEDGER_*), and joins the ledger with lib/context-bill.ts buildBill. bin/gstack-inventory writes docs/inventory.md, --check validates (exit 1 with codes), --json prints. Paid dispatches per run for /ship and /autoplan are quoted from their own docs and found by line, so a moved citation fails --check (INVENTORY_SOURCE_MISSING). Every current row is grandfathered and seeded from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json): in-flight fair tests and proposed registry rows as plans, earlier unfair results under prior with why (stub, unmasked, ceiling, ...), five safety rows. gstack-skill-bill prints the skill's EVIDENCE line. Evidence: plan none (owner ruling R56, GST-914); the ledger itself is the measurement record Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a26dfe9940
commit
37f515bae0
8 files changed
+1208
-4
No files matched your search
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
// gstack-inventory — every gstack part, its load per run and its evidence (ETHOS.md: 9. Earn Its Place). Logic: lib/inventory.ts.
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { buildInventory, checkInventory, DOC_PATH, LEDGER_PATH, renderInventory } from "../lib/inventory";
|
||||
|
||||
const USAGE = `Usage: gstack-inventory [--root <gstack checkout>] [--check | --json]
|
||||
gstack-inventory --help
|
||||
|
||||
Joins the evidence ledger (${LEDGER_PATH}) with gstack-context-bill's sizes and writes ${DOC_PATH}:
|
||||
parts ranked by load per run, each part's evidence status, a "measure next" order (largest untested
|
||||
first), shared-part sizes, paid dispatches per run for /ship and /autoplan, and the key list.
|
||||
--check validates the ledger (every key has one row; each row has its status's fields), the quoted
|
||||
dispatch sources and the shared-part files, and writes nothing. --json prints the inventory.
|
||||
Exit 0 when clean; 1 with one "<key>: <problem> (<CODE>)" line per problem; 2 on a bad argument.`;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes("--help") || args.includes("-h")) { console.log(USAGE); process.exit(0); }
|
||||
const rootAt = args.indexOf("--root");
|
||||
const root = resolve(rootAt >= 0 ? args[rootAt + 1] ?? "" : join(import.meta.dir, ".."));
|
||||
const unknown = args.find((a, i) => !["--check", "--json", "--root"].includes(a) && !(rootAt >= 0 && i === rootAt + 1));
|
||||
if (unknown !== undefined || (rootAt >= 0 && !args[rootAt + 1]) || (args.includes("--check") && args.includes("--json"))) {
|
||||
console.error(unknown !== undefined ? `gstack-inventory: unknown argument '${unknown}'; valid: --root <dir> --check --json --help` : USAGE);
|
||||
process.exit(2);
|
||||
}
|
||||
const report = (problems: { key: string; code: string; message: string }[]) => {
|
||||
for (const p of problems) console.error(`gstack-inventory: ${p.key}: ${p.message} (${p.code})`);
|
||||
};
|
||||
if (args.includes("--check")) {
|
||||
const checked = checkInventory(root);
|
||||
report(checked.problems);
|
||||
if (checked.problems.length) process.exit(1);
|
||||
console.log(`INVENTORY: ok keys=${checked.keys.length} ledger=${LEDGER_PATH} dispatch_sources=${checked.dispatches.length}`);
|
||||
process.exit(0);
|
||||
}
|
||||
const inv = buildInventory(root);
|
||||
report(inv.problems);
|
||||
if (args.includes("--json")) { console.log(JSON.stringify(inv, null, 2)); process.exit(inv.problems.length ? 1 : 0); }
|
||||
if (inv.problems.length) { console.error(`gstack-inventory: ${DOC_PATH} not written; fix the problems above, then rerun`); process.exit(1); }
|
||||
writeFileSync(join(root, DOC_PATH), renderInventory(inv));
|
||||
console.log(`INVENTORY: wrote ${DOC_PATH} keys=${inv.keys.length} commit=${inv.commit}`);
|
||||
@@ -11,7 +11,8 @@ Names the skill's kind: gstack-checkout (edit SKILL.md.tmpl, then bun run gen:sk
|
||||
gstack-install (never edit; gstack-upgrade overwrites it) or user. BASELINE is the newest commit
|
||||
with a "Skill-Diet: <name>" trailer; DUE means the authored file grew over 10% since then.
|
||||
In a gstack checkout, EVAL TASKS names test/fixtures/skill-evals/<name>/tasks.json (or none) and
|
||||
EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format.
|
||||
EVAL_TASKS_INVALID flags a file that breaks docs/designs/SKILL_EVAL_CONTRACT.md's task format, and
|
||||
EVIDENCE prints the skill's row in docs/inventory/evidence.json: its status and source, plan or reason.
|
||||
A name must match exactly; pass a path when it names more than one skill.
|
||||
Exit 0 on a report, 2 on SKILL_NOT_FOUND, SKILL_AMBIGUOUS or SKILL_MD_MISSING (each with a fix line).`;
|
||||
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.",
|
||||
"seed_source": "org-gstack-gbrain/research/gstack-part-evidence.json (GEVA-156 for GST-915, 2026-11-02, gstack-evals main 409ff47)",
|
||||
"parts": {
|
||||
"autoplan": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-3 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: autoplan: untested (never measured)"
|
||||
},
|
||||
"benchmark": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: benchmark: untested (never measured)"
|
||||
},
|
||||
"benchmark-models": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: benchmark-models: untested (never measured)"
|
||||
},
|
||||
"browse": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: browse: untested (never measured)"
|
||||
},
|
||||
"canary": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: canary: untested (never measured)"
|
||||
},
|
||||
"captain": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: captain: untested (never measured)"
|
||||
},
|
||||
"careful": {
|
||||
"status": "safety",
|
||||
"grandfathered": true,
|
||||
"reason": "safety guard: warns before destructive commands (rm -rf, DROP TABLE, force-push); kept on that reason, an average score cannot see the rare bad run it stops",
|
||||
"seed": "GEVA-2, 2026-11-02: careful: exempt (safety guard)"
|
||||
},
|
||||
"claude-code": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: claude-code: untested (never measured)"
|
||||
},
|
||||
"codex": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "not in GEVA-2's table (2026-11-02); never measured"
|
||||
},
|
||||
"context-restore": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: context-restore: untested (never measured)"
|
||||
},
|
||||
"context-save": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: context-save: untested (never measured)"
|
||||
},
|
||||
"cso": {
|
||||
"status": "safety",
|
||||
"grandfathered": true,
|
||||
"reason": "security: supported security findings with explicit coverage; kept on its security reason (GEVA-2 marks it exempt; GStack Minder may reclassify it)",
|
||||
"seed": "GEVA-2, 2026-11-02: cso: exempt (security audit skill, treated as exempt; GStack Minder may reclassify)"
|
||||
},
|
||||
"design-consultation": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-10 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: design-consultation: untested (never measured)"
|
||||
},
|
||||
"design-html": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: design-html: untested (never measured)"
|
||||
},
|
||||
"design-motion": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: design-motion: untested (never measured)"
|
||||
},
|
||||
"design-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-9 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: design-review: untested (never measured)"
|
||||
},
|
||||
"design-shotgun": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: design-shotgun: untested (never measured)"
|
||||
},
|
||||
"deslop-shared-libs": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: deslop-shared-libs: untested (never measured)"
|
||||
},
|
||||
"devex-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: devex-review: untested (never measured)"
|
||||
},
|
||||
"diagram": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: diagram: untested (never measured)"
|
||||
},
|
||||
"document-generate": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: document-generate: untested (never measured)"
|
||||
},
|
||||
"document-release": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: document-release: untested (never measured)"
|
||||
},
|
||||
"eval-plan": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: eval-plan: untested (never measured)"
|
||||
},
|
||||
"freeze": {
|
||||
"status": "safety",
|
||||
"grandfathered": true,
|
||||
"reason": "safety guard: hard-blocks edits outside one directory; kept on that reason",
|
||||
"seed": "GEVA-2, 2026-11-02: freeze: exempt (safety guard)"
|
||||
},
|
||||
"gstack": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: gstack (root skill): untested (never measured)"
|
||||
},
|
||||
"gstack-upgrade": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: upgrade: untested (never measured)"
|
||||
},
|
||||
"guard": {
|
||||
"status": "safety",
|
||||
"grandfathered": true,
|
||||
"reason": "safety guard: careful and freeze together; kept on that reason",
|
||||
"seed": "GEVA-2, 2026-11-02: guard: exempt (safety guard)"
|
||||
},
|
||||
"health": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: health: untested (never measured)"
|
||||
},
|
||||
"investigate": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "R3 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-101-r48-follow-ups-prereg (#112, main f0b81c0) and its amendment (open PR #133), told /investigate vs no gstack on SWE-bench Verified, preset, Opus 5.5 then GPT-6.1 Sol",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "gstack-evals #109 stacks round 1 TB2 (GEVA-89, GEVA-93), wiki/board/2026-11-01-geva-93-stacks-round-1-tb2-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-exposure",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "Opus +3 points [-3, +9] at 1.9x cost; debug report in 0 of 111 trials; particle-1 -5 [-14, +1]"
|
||||
},
|
||||
{
|
||||
"ref": "GSTA-12 Lane B quick look 2, wiki/board/2026-10-29-gsta-12-laneb-quick-2-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "Opus gstack minus none -7 [-13, -2] at 1.7x cost; did not replicate"
|
||||
},
|
||||
{
|
||||
"ref": "GSTA-30 Lane B quick look 3 (#89), wiki/board/2026-10-30-gsta-30-laneb-quick-3-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"small-tier"
|
||||
],
|
||||
"note": "gstack arms on Sonnet, particle-1, DeepSeek and Haiku; Opus ran the none arm only"
|
||||
},
|
||||
{
|
||||
"ref": "HARN-14 replication (harness-evals #181, unmerged)",
|
||||
"why": [
|
||||
"stub",
|
||||
"small-tier"
|
||||
],
|
||||
"note": "Haiku 5.5: gstack minus none +3.5 [-12.3, +20.0]"
|
||||
}
|
||||
],
|
||||
"r48": {
|
||||
"verdict": "FIX-METHOD",
|
||||
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R3 is the fix"
|
||||
},
|
||||
"seed": "GEVA-2, 2026-11-02: investigate: measurement under way (R3)"
|
||||
},
|
||||
"ios-clean": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ios-clean: untested (never measured)"
|
||||
},
|
||||
"ios-design-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ios-design-review: untested (never measured)"
|
||||
},
|
||||
"ios-fix": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ios-fix: untested (never measured)"
|
||||
},
|
||||
"ios-qa": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ios-qa: untested (never measured)"
|
||||
},
|
||||
"ios-sync": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ios-sync: untested (never measured)"
|
||||
},
|
||||
"land-and-deploy": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-8 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: land-and-deploy: untested (never measured)"
|
||||
},
|
||||
"landing-report": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: landing-report: untested (never measured)"
|
||||
},
|
||||
"learn": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "GSTA-13, GEVA-54 and GEVA-65 Lane A runs (wiki/board/2026-10-28-gsta-13-quick-look-3-results and the GEVA-54 and GEVA-65 posts)",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-true-no-gstack-arm",
|
||||
"ceiling"
|
||||
],
|
||||
"note": "Opus contract tasks 27/30 vs 27/30"
|
||||
},
|
||||
{
|
||||
"ref": "GEVA-65 Lane A rerun row 5 (GPT-6.1 Sol in Codex)",
|
||||
"why": [
|
||||
"not-claude-code",
|
||||
"no-true-no-gstack-arm"
|
||||
],
|
||||
"note": "Sol failed the no-code-change safety task in 5 of 9 skill trials vs 0 of 3 without; small n, a lead to check, not a hurts verdict"
|
||||
}
|
||||
],
|
||||
"seed": "GEVA-2, 2026-11-02: learn: untested (unfair test: SDK stub; no true no-gstack arm; Opus at ceiling; Sol ran in Codex, not Claude Code)"
|
||||
},
|
||||
"make-pdf": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: make-pdf: untested (never measured)"
|
||||
},
|
||||
"modernize": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: modernize: untested (never measured)"
|
||||
},
|
||||
"office-hours": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "GSTA-13 quick look 3 Lane A, wiki/board/2026-10-28-gsta-13-quick-look-3-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-true-no-gstack-arm"
|
||||
],
|
||||
"note": "contract tasks on Opus: gstack 74% vs none 28%; the tasks grade the skill's own contract, a manipulation check, not lift"
|
||||
},
|
||||
{
|
||||
"ref": "GEVA-65 Lane A rerun, wiki/board/2026-10-31-geva-65-lanea-shakedown-rerun-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-true-no-gstack-arm"
|
||||
],
|
||||
"note": "Opus before 0.67, after 0.76, none 0.36"
|
||||
}
|
||||
],
|
||||
"seed": "GEVA-2, 2026-11-02: office-hours: untested (unfair test: SDK stub; no true no-gstack arm; tasks grade the skill's own contract, a manipulation check, not lift)"
|
||||
},
|
||||
"open-gstack-browser": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: open-gstack-browser: untested (never measured)"
|
||||
},
|
||||
"pair-agent": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: pair-agent: untested (never measured)"
|
||||
},
|
||||
"plan-ceo-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-5 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "CEO panel v2 admission (GEVA-142), wiki/board/2026-11-02-geva-142-ceo-panel-v2-admission-results (open PR #131)",
|
||||
"why": [
|
||||
"ceiling",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "preset, Opus 5.5 only: must-fix recall 1.00 in all three arms; plain invocation ran the procedure in 0 of 16"
|
||||
},
|
||||
{
|
||||
"ref": "Planning R1a pilot (GEVA-102), wiki/board/2026-11-01-geva-102-planning-r1a-pilot-stop (#114)",
|
||||
"why": [
|
||||
"stub",
|
||||
"ceiling",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "none-opus 8/8; gstack-opus read the sections file in 0 of 4"
|
||||
}
|
||||
],
|
||||
"r48": {
|
||||
"verdict": "FIX-METHOD",
|
||||
"ref": "CEO panel v2 admission (GEVA-142, open PR #131): FIX-METHOD on the panel, a harder panel is planned"
|
||||
},
|
||||
"seed": "GEVA-2, 2026-11-02: plan-ceo-review: untested (unfair test: ceiling, control 12 of 12 so no MDD; plain invocation ran the procedure in 0 of 16)"
|
||||
},
|
||||
"plan-design-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-7 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: plan-design-review: untested (never measured)"
|
||||
},
|
||||
"plan-devex-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-6 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: plan-devex-review: untested (never measured)"
|
||||
},
|
||||
"plan-eng-review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-2 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: plan-eng-review: untested (never measured)"
|
||||
},
|
||||
"plan-tune": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: plan-tune: untested (never measured)"
|
||||
},
|
||||
"qa": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: qa: untested (never measured)"
|
||||
},
|
||||
"qa-only": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: qa-only: untested (never measured)"
|
||||
},
|
||||
"retro": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: retro: untested (never measured)"
|
||||
},
|
||||
"review": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "gstack-evals #108 stacks round 1 review (GEVA-89), wiki/board/2026-11-01-geva-89-stacks-round-1-review-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-exposure",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "Opus 5.5 blocking recall 9.5% vs 7.1%, +2.4 points [-6.4, +11.1]; specialists and adversarial pass ran in 0 of 66 sessions"
|
||||
},
|
||||
{
|
||||
"ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "gstack-opus minus none-opus +3.5 points [-9.6, +16.7]"
|
||||
},
|
||||
{
|
||||
"ref": "GSTA-11 quick look 1 review replay, wiki/board/2026-10-28-gsta-11-review-replay-results",
|
||||
"why": [
|
||||
"not-claude-code",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "/review's prompts through the Messages API; Opus recall 44% vs 39% plain, 12 bugs"
|
||||
}
|
||||
],
|
||||
"r48": {
|
||||
"verdict": "FIX-METHOD",
|
||||
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R2 is the fix"
|
||||
},
|
||||
"seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)"
|
||||
},
|
||||
"scrape": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: scrape: untested (never measured)"
|
||||
},
|
||||
"setup-browser-cookies": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: setup-browser-cookies: untested (never measured)"
|
||||
},
|
||||
"setup-deploy": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: setup-deploy: untested (never measured)"
|
||||
},
|
||||
"setup-gbrain": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: setup-gbrain: untested (never measured)"
|
||||
},
|
||||
"ship": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-4 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"seed": "GEVA-2, 2026-11-02: ship: untested (never measured)"
|
||||
},
|
||||
"skillify": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: skillify: untested (never measured)"
|
||||
},
|
||||
"spec": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: spec: untested (never measured)"
|
||||
},
|
||||
"sync-gbrain": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: sync-gbrain: untested (never measured)"
|
||||
},
|
||||
"test-audit": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: test-audit: untested (never measured)"
|
||||
},
|
||||
"ubicloud": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: ubicloud: untested (never measured)"
|
||||
},
|
||||
"unfreeze": {
|
||||
"status": "safety",
|
||||
"grandfathered": true,
|
||||
"reason": "safety guard: the off switch for freeze's edit restriction; kept with the guard it releases",
|
||||
"seed": "GEVA-2, 2026-11-02: unfreeze: exempt (safety guard (undoes freeze))"
|
||||
},
|
||||
"verify": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "not in GEVA-2's table (2026-11-02); never measured"
|
||||
},
|
||||
"write-skill": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: write-skill: untested (never measured)"
|
||||
},
|
||||
"preamble": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "I1 ceremony cut (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-108-ceremony-cut-prereg (#122, main 409ff47) with its G3 amendment (#128), preset, Opus 5.5 and particle-1",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "GEVA-100 exposure counts, org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100)",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-exposure"
|
||||
],
|
||||
"note": "ceremony sections ran in 0 of 66 Opus review sessions and 6 of 111 Opus TB2 trials; no effect estimate"
|
||||
}
|
||||
],
|
||||
"r48": {
|
||||
"verdict": "CUT-CANDIDATE",
|
||||
"ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); taken by Garry as the I1 ceremony-cut eval"
|
||||
},
|
||||
"seed": "GEVA-2, 2026-11-02: preamble (ceremony): measurement under way (I1, ceremony-cut eval)"
|
||||
},
|
||||
"catalog": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "Run D (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-01-geva-125-untold-discovery-prereg (#125, main 9820b14), untold gstack vs placebo vs no gstack, preset vs stub, Opus 5.5, GPT-6.1 Sol and particle-1; also measures /investigate's description",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "GSTA-12 Lane B quick look 1, wiki/board/2026-10-29-gsta-12-laneb-quick-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"no-exposure",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "untold skill opened in 0 of 180 gstack-arm trials; Opus gstack minus none +5 [-3, +17]"
|
||||
}
|
||||
],
|
||||
"seed": "GEVA-2, 2026-11-02: always-on skill catalog (descriptions): measurement under way (Run D)"
|
||||
},
|
||||
"capy-skill": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: capy/skill/gstack/SKILL.md (Capy context): untested (never measured)",
|
||||
"note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it"
|
||||
},
|
||||
"capy-factory": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "none",
|
||||
"seed": "GEVA-2, 2026-11-02: capy/skill/gstack/factory.md (Capy context): untested (never measured)",
|
||||
"note": "loads only in Capy sessions, which the claude_code preset fair test cannot reach yet; needs a Captain call on how to test it"
|
||||
},
|
||||
"ship-review-panel": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "GEVA-2 registry row R56-1 (org-gstack-gbrain/research/gstack-part-evidence.json registry_rows; proposed, not yet preregistered)",
|
||||
"prior": [
|
||||
{
|
||||
"ref": "gstack-evals #120 R1 blind re-judge (GEVA-113), wiki/board/2026-11-01-geva-113-r1-blind-rejudge-results",
|
||||
"why": [
|
||||
"stub",
|
||||
"not-claude-code",
|
||||
"unmasked"
|
||||
],
|
||||
"note": "arm C minus none +56.9 points [40.5, 72.2]; the none arm ran on the stub, arm C in Capy at another gstack version, on a cohort mined from gstack's own catches"
|
||||
},
|
||||
{
|
||||
"ref": "GSTA-48 review no-loss arm D (#77), wiki/board/2026-10-30-gsta-48-review-noloss-arm-how-to",
|
||||
"why": [
|
||||
"variant-only",
|
||||
"not-claude-code"
|
||||
],
|
||||
"note": "arm D vs pooled arm C, no no-gstack arm"
|
||||
}
|
||||
],
|
||||
"seed": "GEVA-2, 2026-11-02: ship: review panel (specialist, adversarial, targeted, outside voice, red team): untested (unfair test: none arm on the SDK stub; arms from different harnesses and gstack versions; cohort mined from gstack's own catches)"
|
||||
},
|
||||
"ship-docs-audit": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "folded into GEVA-2 registry row R56-4 (ship); proposed, not yet preregistered",
|
||||
"seed": "GEVA-2, 2026-11-02: ship: docs audit (Step 14.5): untested (never measured)"
|
||||
},
|
||||
"autoplan-phases": {
|
||||
"status": "untested",
|
||||
"grandfathered": true,
|
||||
"plan": "folded into GEVA-2 registry row R56-3 (autoplan); proposed, not yet preregistered",
|
||||
"seed": "GEVA-2, 2026-11-02: autoplan: CEO, design, DX and eng phases, outside voice: untested (never measured)"
|
||||
}
|
||||
},
|
||||
"unmapped": [
|
||||
{
|
||||
"row": "review: Review Army and adversarial pass",
|
||||
"why": "a section inside /review, not its own key; R2 measures it as part of the review row's plan"
|
||||
},
|
||||
{
|
||||
"row": "ship: QA probes (Step 9.2.1)",
|
||||
"why": "a step inside /ship with no separate size; folded into the ship row (GEVA-2 registry row R56-4)"
|
||||
},
|
||||
{
|
||||
"row": "gstack as a whole (installed, any skill)",
|
||||
"why": "not a single text part; GEVA-2's real-world outcome lane measures it (gstack-evals plan #111), not a measurement plan for any one part"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
/**
|
||||
* inventory — every part of gstack, its load per run and its evidence (ETHOS.md: 9. Earn Its Place).
|
||||
*
|
||||
* Keys are the skill directories discoverTemplates() finds (the root skill as `gstack`) plus
|
||||
* SHARED_KEYS. The ledger, docs/inventory/evidence.json, is hand-kept: each row is checked by its
|
||||
* status's own fields (validateEntry). buildInventory joins the ledger with lib/context-bill.ts
|
||||
* buildBill (offline token estimates) and the dispatch counts ship and autoplan already document,
|
||||
* each found by its quoted line so a citation that drifts is named. Read-only except the CLI's write.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { discoverTemplates } from '../scripts/discover-skills';
|
||||
import { buildBill, contentClass, TOKEN_DIVISOR, TOKEN_DIVISORS } from './context-bill';
|
||||
|
||||
export const LEDGER_PATH = 'docs/inventory/evidence.json';
|
||||
export const DOC_PATH = 'docs/inventory.md';
|
||||
export const ROOT_KEY = 'gstack';
|
||||
|
||||
export interface SharedPart { key: string; label: string; load: string; files: string[] }
|
||||
/** Shared parts: sized from their files (catalog from the bill's frontmatter). */
|
||||
export const SHARED_PARTS: SharedPart[] = [
|
||||
{ key: 'preamble', label: 'shared preamble (sections/preamble.md, identical in every tier 2+ skill)', load: 'every tier 2+ skill run, inside that skill\'s load', files: ['ship/sections/preamble.md'] },
|
||||
{ key: 'catalog', label: 'skill catalog (every skill\'s frontmatter)', load: 'every session', files: [] },
|
||||
{ key: 'capy-skill', label: 'capy/skill/gstack/SKILL.md', load: 'every Capy session', files: ['capy/skill/gstack/SKILL.md'] },
|
||||
{ key: 'capy-factory', label: 'capy/skill/gstack/factory.md', load: 'every Capy session', files: ['capy/skill/gstack/factory.md'] },
|
||||
{ key: 'ship-review-panel', label: '/ship review panel (review-army and adversarial sections)', load: 'every /ship run, by depth', files: ['ship/sections/review-army.md', 'ship/sections/adversarial.md'] },
|
||||
{ key: 'ship-docs-audit', label: '/ship docs audit (Step 14.5)', load: 'every /ship run', files: ['ship/sections/documentation.md'] },
|
||||
{ key: 'autoplan-phases', label: '/autoplan CEO, design, DX and eng phases', load: 'every /autoplan run, by scope', files: ['autoplan/sections/ceo-phase.md', 'autoplan/sections/design-phase.md', 'autoplan/sections/dx-phase.md', 'autoplan/sections/eng-phase.md'] },
|
||||
];
|
||||
export const SHARED_KEYS = SHARED_PARTS.map(p => p.key);
|
||||
|
||||
export interface Dispatch { skill: 'ship' | 'autoplan'; what: string; count: string; file: string; quote: string }
|
||||
/** Paid dispatches per run, from the budget tables and docs that already state them (no new instrumentation). */
|
||||
export const DISPATCHES: Dispatch[] = [
|
||||
{ skill: 'ship', what: 'Step 9 review, pass 1, light depth, docs or tests only', count: '1 reader (the core checklist)', file: 'docs/review-policy.md', quote: '| **light**, runtime 0 |' },
|
||||
{ skill: 'ship', what: 'Step 9 review, pass 1, light depth, code', count: '3 readers (core, one batched specialist, native adversarial), plus the outside voice on a diff that publishes numbers', file: 'docs/review-policy.md', quote: '| **light**, runtime above 0 |' },
|
||||
{ skill: 'ship', what: 'Step 9 review, pass 1, standard depth', count: 'the batched specialist, the Red Team at 200 runtime lines, adversarial and outside voice', file: 'docs/review-policy.md', quote: '| **standard** |' },
|
||||
{ skill: 'ship', what: 'Step 9 review, pass 1, full depth', count: 'the batched specialist on the whole diff and the Red Team always', file: 'docs/review-policy.md', quote: '| **full** |' },
|
||||
{ skill: 'ship', what: 'targeted reader on money, gate, auth or secret code', count: '+1 at every depth', file: 'docs/review-policy.md', quote: '`gstack-review-plan` dispatches one' },
|
||||
{ skill: 'ship', what: 'audits (Steps 7, 8 and 14.5)', count: '3', file: 'lib/ship-budget-calibration.json', quote: 'each audit dispatch (Steps 7, 8, 14.5)' },
|
||||
{ skill: 'ship', what: 'measured: one /ship child (gstack-evals#31)', count: '10 review dispatches', file: 'lib/ship-budget-calibration.json', quote: 'its 10 review dispatches' },
|
||||
{ skill: 'ship', what: 'prior cost of one dispatch on the Capy meter', count: '7,430,000 tokens', file: 'lib/ship-budget-calibration.json', quote: '"per_dispatch": 7430000' },
|
||||
{ skill: 'autoplan', what: 'voices per phase (native and outside)', count: '2', file: 'autoplan/DRIVER.md.tmpl', quote: 'per voice. Dispatch the native prompt to a subagent on your model and the outside prompt to' },
|
||||
{ skill: 'autoplan', what: 'phases per round: CEO and Eng always, Design with UI scope, DX with developer-facing scope', count: '2 to 4, so 4 to 8 voice dispatches a round', file: 'autoplan/SKILL.md.tmpl', quote: 'Phases MUST run in strict order: CEO → Design (if UI scope) → DX (if developer-facing' },
|
||||
{ skill: 'autoplan', what: 'CEO spec review loop', count: 'up to 3 launches', file: 'autoplan/sections/ceo-phase.md.tmpl', quote: 'three-launch cap' },
|
||||
{ skill: 'autoplan', what: 'review rounds per phase (new unattended runs)', count: 'up to 2', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_MAX_ROUNDS = 2;' },
|
||||
{ skill: 'autoplan', what: 'reserved per voice on Capy or under a spend cap', count: '$5', file: 'lib/autoplan-budget.ts', quote: 'export const DEFAULT_HOST_ESTIMATE_USD = 5;' },
|
||||
];
|
||||
|
||||
export const STATUSES = ['helps', 'no-effect', 'hurts', 'untested', 'safety'] as const;
|
||||
export type Status = (typeof STATUSES)[number];
|
||||
export const R48_VERDICTS = ['KEEP', 'FIX-METHOD', 'CUT-CANDIDATE'] as const;
|
||||
/** R54: the frontier models under test; a row may add at most one named low-cost primary, never Haiku or Sonnet. */
|
||||
export const FRONTIER_MODELS = ['claude-opus-5-5', 'gpt-6.1-sol'];
|
||||
/** Why an earlier result does not count as fair evidence yet. */
|
||||
export const PRIOR_WHY: Record<string, string> = {
|
||||
stub: 'ran on the Agent SDK stub prompt, not the claude_code preset',
|
||||
'small-tier': 'Haiku, Sonnet or another non-frontier model only',
|
||||
unmasked: 'no mask recorded: the no-gstack arm predates the masking rule and was not re-checked',
|
||||
ceiling: 'control at the ceiling, so no difference could show',
|
||||
'no-true-no-gstack-arm': 'the none arm still had gstack installed',
|
||||
'no-exposure': 'the model never used the part',
|
||||
'not-claude-code': 'ran outside Claude Code (Messages API, Codex or Capy)',
|
||||
'variant-only': 'variant against variant, no no-gstack arm',
|
||||
};
|
||||
const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'seed', 'note'];
|
||||
|
||||
export type LedgerCode =
|
||||
| 'LEDGER_PARSE' | 'LEDGER_KEY_MISSING' | 'LEDGER_KEY_EXTRA' | 'LEDGER_STATUS_INVALID' | 'LEDGER_FIELD_UNKNOWN'
|
||||
| 'LEDGER_SOURCES_MISSING' | 'LEDGER_MODELS_NOT_FRONTIER' | 'LEDGER_PROMPT_NOT_PRESET' | 'LEDGER_SCOPE_MISSING'
|
||||
| 'LEDGER_UNMASKED' | 'LEDGER_R48_MISSING' | 'LEDGER_PLAN_MISSING' | 'LEDGER_PLAN_NONE_NEW' | 'LEDGER_PLAN_NOT_CONCRETE'
|
||||
| 'LEDGER_REASON_MISSING' | 'LEDGER_OWNER_MISSING' | 'LEDGER_MEASURE_BY_MISSING' | 'LEDGER_PRIOR_INVALID'
|
||||
| 'INVENTORY_SOURCE_MISSING' | 'INVENTORY_PART_MISSING';
|
||||
export interface LedgerProblem { key: string; code: LedgerCode; message: string }
|
||||
export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; seed?: string; note?: string }
|
||||
export interface Ledger { parts: Record<string, LedgerEntry>; unmapped?: Array<{ row: string; why: string }>; [k: string]: unknown }
|
||||
|
||||
const isObj = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v);
|
||||
const str = (v: unknown): v is string => typeof v === 'string' && v.trim() !== '';
|
||||
const validR48 = (v: unknown) => isObj(v) && (R48_VERDICTS as readonly string[]).includes(v.verdict as string) && str(v.ref);
|
||||
|
||||
function frontierModels(v: unknown): boolean {
|
||||
if (!Array.isArray(v) || !v.length || !v.every(str)) return false;
|
||||
const ids = v.map(m => m.slice(m.lastIndexOf('/') + 1));
|
||||
const other = ids.filter(m => !FRONTIER_MODELS.includes(m));
|
||||
return ids.length > other.length && other.length <= 1 && !other.some(m => /haiku|sonnet/i.test(m));
|
||||
}
|
||||
|
||||
/** Problems with one ledger row, by its status's own fields (helps, no-effect, hurts, untested, safety). */
|
||||
export function validateEntry(key: string, entry: unknown): LedgerProblem[] {
|
||||
const problems: LedgerProblem[] = [];
|
||||
const add = (code: LedgerCode, message: string) => problems.push({ key, code, message });
|
||||
if (!isObj(entry)) { add('LEDGER_STATUS_INVALID', 'the row is not an object'); return problems; }
|
||||
for (const f of Object.keys(entry)) if (!FIELDS.includes(f)) add('LEDGER_FIELD_UNKNOWN', `unknown field ${f}; fields: ${FIELDS.join(', ')}`);
|
||||
const prior = entry.prior;
|
||||
if (prior !== undefined && !(Array.isArray(prior) && prior.every(p => isObj(p) && str(p.ref) && Array.isArray(p.why) && p.why.length > 0 && p.why.every(w => typeof w === 'string' && w in PRIOR_WHY))))
|
||||
add('LEDGER_PRIOR_INVALID', `prior needs [{ref, why: [...]}] with why from: ${Object.keys(PRIOR_WHY).join(', ')}`);
|
||||
const status = entry.status;
|
||||
if (status === 'helps' || status === 'no-effect' || status === 'hurts') {
|
||||
if (!Array.isArray(entry.sources) || !entry.sources.length || !entry.sources.every(str)) add('LEDGER_SOURCES_MISSING', `${status} needs sources: one or more run ids or results files`);
|
||||
if (!frontierModels(entry.models)) add('LEDGER_MODELS_NOT_FRONTIER', `models must include ${FRONTIER_MODELS.join(' or ')} plus at most one named low-cost primary, never Haiku or Sonnet (R54)`);
|
||||
if (entry.prompt !== 'preset') add('LEDGER_PROMPT_NOT_PRESET', 'prompt must be "preset" (the claude_code system prompt, not the Agent SDK stub)');
|
||||
if (!str(entry.scope)) add('LEDGER_SCOPE_MISSING', 'scope must say what was measured');
|
||||
if (entry.masked !== true) add('LEDGER_UNMASKED', 'masked must be true: the no-gstack arm was masked from the Capy drives and gstack installs');
|
||||
if (status !== 'helps' && !validR48(entry.r48)) add('LEDGER_R48_MISSING', `${status} needs r48: {verdict: ${R48_VERDICTS.join('|')}, ref}`);
|
||||
} else if (status === 'untested') {
|
||||
const grandfathered = entry.grandfathered === true;
|
||||
if (!str(entry.plan)) add('LEDGER_PLAN_MISSING', 'untested needs plan: a reference, or "none" on a grandfathered row');
|
||||
else if (entry.plan === 'none') { if (!grandfathered) add('LEDGER_PLAN_NONE_NEW', 'plan: none is only for grandfathered rows; a new part names its measurement plan'); }
|
||||
else if (!grandfathered && !/tasks\.json$|prereg/i.test(entry.plan)) add('LEDGER_PLAN_NOT_CONCRETE', 'a new part\'s plan is a test/fixtures/skill-evals/<name>/tasks.json or an /eval-plan preregistration');
|
||||
if (!grandfathered) {
|
||||
if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'a new untested part states its reason');
|
||||
if (!str(entry.owner)) add('LEDGER_OWNER_MISSING', 'a new untested part names its owner');
|
||||
if (!str(entry.measure_by) || !/^\d{4}-\d{2}-\d{2}$/.test(entry.measure_by) || Number.isNaN(Date.parse(entry.measure_by))) add('LEDGER_MEASURE_BY_MISSING', 'a new untested part needs measure_by: YYYY-MM-DD');
|
||||
}
|
||||
} else if (status === 'safety') {
|
||||
if (!str(entry.reason)) add('LEDGER_REASON_MISSING', 'safety needs reason: what the guard protects');
|
||||
} else add('LEDGER_STATUS_INVALID', `status must be one of ${STATUSES.join(', ')}`);
|
||||
if (entry.r48 !== undefined && !validR48(entry.r48) && (status === 'untested' || status === 'safety')) add('LEDGER_R48_MISSING', `r48 needs {verdict: ${R48_VERDICTS.join('|')}, ref}`);
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Coverage (no missing, no extra key) then every row's own schema. */
|
||||
export function validateLedger(ledger: unknown, keys: string[]): LedgerProblem[] {
|
||||
if (!isObj(ledger) || !isObj(ledger.parts)) return [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `${LEDGER_PATH} needs a parts object` }];
|
||||
const parts = ledger.parts;
|
||||
return [
|
||||
...keys.filter(k => !(k in parts)).map(k => ({ key: k, code: 'LEDGER_KEY_MISSING' as const, message: `no row in ${LEDGER_PATH}` })),
|
||||
...Object.keys(parts).filter(k => !keys.includes(k)).map(k => ({ key: k, code: 'LEDGER_KEY_EXTRA' as const, message: 'not an inventory key (the key list is in docs/inventory.md)' })),
|
||||
...keys.filter(k => k in parts).flatMap(k => validateEntry(k, parts[k])),
|
||||
];
|
||||
}
|
||||
|
||||
export function loadLedger(root: string): { ledger: Ledger | null; problems: LedgerProblem[] } {
|
||||
try {
|
||||
return { ledger: JSON.parse(fs.readFileSync(path.join(root, LEDGER_PATH), 'utf8')), problems: [] };
|
||||
} catch (e) {
|
||||
return { ledger: null, problems: [{ key: '(ledger)', code: 'LEDGER_PARSE', message: `cannot read ${LEDGER_PATH}: ${(e as Error).message}` }] };
|
||||
}
|
||||
}
|
||||
|
||||
/** The skill keys (discoverTemplates, root skill as gstack, symlinked folders excluded) then SHARED_KEYS. */
|
||||
export function inventoryKeys(root: string): string[] {
|
||||
const skills = discoverTemplates(root).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d));
|
||||
return [...skills.sort(), ...SHARED_KEYS];
|
||||
}
|
||||
|
||||
/** Installed skill name to key: gstack-<name> is <name>; gstack-upgrade keeps its name. */
|
||||
export const keyOf = (name: string) => (name !== 'gstack-upgrade' && name.startsWith('gstack-') ? name.slice('gstack-'.length) : name);
|
||||
|
||||
/** The status's evidence in one phrase: sources, plan or reason. */
|
||||
export function evidenceRef(e: LedgerEntry): string {
|
||||
if (e.status === 'untested') return `plan: ${e.plan}${e.grandfathered ? ' (grandfathered)' : ` (owner ${e.owner}, measure by ${e.measure_by})`}`;
|
||||
if (e.status === 'safety') return `reason: ${e.reason}`;
|
||||
return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}`;
|
||||
}
|
||||
|
||||
/** gstack-skill-bill's EVIDENCE line for one skill folder in a gstack checkout. */
|
||||
export function skillEvidence(root: string, dir: string): { key: string; status: Status | 'missing'; ref: string; line: string } {
|
||||
const key = path.relative(root, dir) || ROOT_KEY;
|
||||
const entry = loadLedger(root).ledger?.parts?.[key];
|
||||
if (!entry) {
|
||||
const ref = `no ${key} row in ${LEDGER_PATH}; add one (ETHOS.md: 9. Earn Its Place)`;
|
||||
return { key, status: 'missing', ref, line: `EVIDENCE: missing: ${ref}` };
|
||||
}
|
||||
const ref = evidenceRef(entry);
|
||||
return { key, status: entry.status, ref, line: `EVIDENCE: ${entry.status} ${ref}` };
|
||||
}
|
||||
|
||||
export interface CitedDispatch extends Dispatch { line: number | null }
|
||||
const tokensOf = (rel: string, bytes: number) => Math.round(bytes / (TOKEN_DIVISORS[contentClass(rel)] ?? TOKEN_DIVISOR));
|
||||
|
||||
/** Ledger, dispatch citations and shared-part files; no bill, so --check stays fast. */
|
||||
export function checkInventory(root: string): { keys: string[]; ledger: Ledger | null; dispatches: CitedDispatch[]; problems: LedgerProblem[] } {
|
||||
const keys = inventoryKeys(root);
|
||||
const loaded = loadLedger(root);
|
||||
const problems = [...loaded.problems, ...(loaded.ledger ? validateLedger(loaded.ledger, keys) : [])];
|
||||
const dispatches = DISPATCHES.map(d => {
|
||||
const file = path.join(root, d.file);
|
||||
const idx = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').findIndex(l => l.includes(d.quote)) : -1;
|
||||
if (idx < 0) problems.push({ key: `dispatch:${d.skill}`, code: 'INVENTORY_SOURCE_MISSING', message: `${d.file} no longer holds "${d.quote}"; update DISPATCHES in lib/inventory.ts` });
|
||||
return { ...d, line: idx < 0 ? null : idx + 1 };
|
||||
});
|
||||
for (const part of SHARED_PARTS) for (const f of part.files) if (!fs.existsSync(path.join(root, f)))
|
||||
problems.push({ key: part.key, code: 'INVENTORY_PART_MISSING', message: `${f} is missing; update SHARED_PARTS in lib/inventory.ts` });
|
||||
return { keys, ledger: loaded.ledger, dispatches, problems };
|
||||
}
|
||||
|
||||
export interface PartRow { key: string; kind: 'skill' | 'shared'; label: string; loads: string; loadTokens: number | null; alwaysOnTokens: number | null; status: Status | 'missing'; ref: string }
|
||||
export interface Inventory {
|
||||
commit: string; date: string; tokenSource: string; keys: string[]; parts: PartRow[];
|
||||
tiers: Array<{ tier: string; skills: string[] }>; dispatches: CitedDispatch[];
|
||||
prior: Array<{ key: string; ref: string; why: string[]; note?: string }>;
|
||||
unmapped: Array<{ row: string; why: string }>; problems: LedgerProblem[];
|
||||
}
|
||||
|
||||
function git(root: string, args: string[]): string | null {
|
||||
const r = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8', timeout: 10_000 });
|
||||
return r.status === 0 ? r.stdout.trim() : null;
|
||||
}
|
||||
|
||||
/** The commit the snapshot measured, its date, and whether the tree had other changes. */
|
||||
function measuredAt(root: string): { commit: string; date: string } {
|
||||
const top = git(root, ['rev-parse', '--show-toplevel']);
|
||||
if (!top || path.resolve(top) !== path.resolve(root)) return { commit: 'none', date: 'none' };
|
||||
const dirty = git(root, ['status', '--porcelain', '--', '.', `:!${DOC_PATH}`]);
|
||||
return { commit: (git(root, ['rev-parse', '--short=9', 'HEAD']) ?? 'none') + (dirty ? ' (with uncommitted changes)' : ''), date: git(root, ['log', '-1', '--format=%cs']) ?? 'none' };
|
||||
}
|
||||
|
||||
/** Joins the ledger with the context bill: one row per key, ranked by load per run. */
|
||||
export function buildInventory(root: string): Inventory {
|
||||
const checked = checkInventory(root);
|
||||
const bill = buildBill(root);
|
||||
const resolved = path.resolve(root);
|
||||
const bySkill = new Map(bill.skills.map(s => [path.relative(resolved, s.dir) || ROOT_KEY, s]));
|
||||
const parts = checked.ledger?.parts ?? {};
|
||||
const skillKeys = checked.keys.filter(k => !SHARED_KEYS.includes(k));
|
||||
const evidence = (key: string) => (parts[key] ? { status: parts[key]!.status, ref: evidenceRef(parts[key]!) } : { status: 'missing' as const, ref: 'no row' });
|
||||
const rows: PartRow[] = skillKeys.map(key => {
|
||||
const s = bySkill.get(key);
|
||||
return { key, kind: 'skill', label: key === ROOT_KEY ? 'root skill (/gstack router)' : `/${key}`, loads: 'every use', loadTokens: s ? Math.round(s.perInvocationTokens) : null, alwaysOnTokens: s ? Math.round(s.frontmatterTokens) : null, ...evidence(key) };
|
||||
});
|
||||
const catalog = skillKeys.reduce((n, k) => n + (bySkill.get(k)?.frontmatterTokens ?? 0), 0);
|
||||
for (const part of SHARED_PARTS) {
|
||||
const sizes = part.files.map(f => { const file = path.join(root, f); return fs.existsSync(file) ? tokensOf(f, fs.statSync(file).size) : 0; });
|
||||
const tokens = part.key === 'catalog' ? Math.round(catalog) : sizes.reduce((a, b) => a + b, 0);
|
||||
rows.push({ key: part.key, kind: 'shared', label: part.label, loads: part.load, loadTokens: tokens, alwaysOnTokens: part.key === 'catalog' ? tokens : null, ...evidence(part.key) });
|
||||
}
|
||||
rows.sort((a, b) => (b.loadTokens ?? -1) - (a.loadTokens ?? -1) || a.key.localeCompare(b.key));
|
||||
const tiers = new Map<string, string[]>();
|
||||
for (const key of skillKeys) {
|
||||
const tmpl = path.join(root, key === ROOT_KEY ? '' : key, 'SKILL.md.tmpl');
|
||||
const tier = /^preamble-tier:\s*(\d+)/m.exec(fs.readFileSync(tmpl, 'utf8'))?.[1] ?? 'none';
|
||||
tiers.set(tier, [...(tiers.get(tier) ?? []), key]);
|
||||
}
|
||||
return {
|
||||
...measuredAt(root),
|
||||
tokenSource: bill.tokenSource,
|
||||
keys: checked.keys,
|
||||
parts: rows,
|
||||
tiers: [...tiers].sort(([a], [b]) => a.localeCompare(b)).map(([tier, skills]) => ({ tier, skills })),
|
||||
dispatches: checked.dispatches,
|
||||
prior: checked.keys.flatMap(key => (parts[key]?.prior ?? []).map(p => ({ key, ...p }))),
|
||||
unmapped: checked.ledger?.unmapped ?? [],
|
||||
problems: checked.problems,
|
||||
};
|
||||
}
|
||||
|
||||
const n = (v: number | null) => (v === null ? 'n/a' : v.toLocaleString('en-US'));
|
||||
const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
||||
|
||||
function renderRanking(inv: Inventory): string[] {
|
||||
const out = ['## Parts by load per run', '', 'Load per run: a skill\'s SKILL.md plus the files it must read on every use (`perInvocationTokens` in lib/context-bill.ts); a shared part\'s files. Always-on: frontmatter every session\'s skill scanner reads.', '',
|
||||
'| Rank | Part | Kind | Load per run (tokens) | Always-on (tokens) | Loads | Status | Source, plan or reason |', '|---|---|---|---|---|---|---|---|'];
|
||||
inv.parts.forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${p.kind} | ${n(p.loadTokens)} | ${n(p.alwaysOnTokens)} | ${cell(p.loads)} | ${p.status} | ${cell(p.ref)} |`));
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderMeasureNext(inv: Inventory): string[] {
|
||||
const out = ['## Measure next', '', 'Untested parts, largest load per run first. A fair test runs the claude_code preset, frontier models (Opus 5.5 and GPT-6.1 Sol, plus at most one named low-cost primary) and a masked no-gstack arm ([TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm)). Safety rows are not on this list.', '',
|
||||
'| Order | Part | Load per run (tokens) | Plan |', '|---|---|---|---|'];
|
||||
inv.parts.filter(p => p.status === 'untested').forEach((p, i) => out.push(`| ${i + 1} | ${p.key} | ${n(p.loadTokens)} | ${cell(p.ref.replace(/^plan: /, ''))} |`));
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderShared(inv: Inventory): string[] {
|
||||
const out = ['## Shared parts and preamble tiers', '', '| Tier | Skills | What loads |', '|---|---|---|'];
|
||||
const what: Record<string, string> = { '1': 'a short preamble inline in SKILL.md', none: 'no preamble' };
|
||||
for (const t of inv.tiers) out.push(`| ${t.tier} | ${t.skills.join(', ')} | ${what[t.tier] ?? 'the shared sections/preamble.md (row preamble), once per session'} |`);
|
||||
out.push('', ...SHARED_PARTS.map(p => `- **${p.key}**: ${p.label}${p.files.length ? ` (${p.files.join(', ')})` : ''}; loads ${p.load}.`));
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderDispatches(inv: Inventory): string[] {
|
||||
const out = ['## Paid dispatches per run', '', 'Each count is quoted from the file and line named; `bin/gstack-inventory --check` fails when a quoted line moves away.', '', '| Skill | What | Count | Source |', '|---|---|---|---|'];
|
||||
for (const d of inv.dispatches) out.push(`| ${d.skill} | ${cell(d.what)} | ${cell(d.count)} | ${d.line ? `${d.file}:${d.line}` : `${d.file}: source not found`} |`);
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderPrior(inv: Inventory): string[] {
|
||||
const out = ['## Earlier results that do not count yet', '', ...Object.entries(PRIOR_WHY).map(([k, v]) => `- \`${k}\`: ${v}.`), '', '| Part | Result | Why it does not count | Note |', '|---|---|---|---|'];
|
||||
for (const p of inv.prior) out.push(`| ${p.key} | ${cell(p.ref)} | ${p.why.join(', ')} | ${cell(p.note ?? '')} |`);
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderKeys(inv: Inventory): string[] {
|
||||
const skills = inv.keys.filter(k => !SHARED_KEYS.includes(k));
|
||||
return ['## Key list and aliases', '', `${inv.keys.length} keys: ${skills.length} skills (the folders \`discoverTemplates()\` finds, the root skill as \`gstack\`) and ${SHARED_KEYS.length} shared parts.`, '',
|
||||
`- Skills: ${skills.join(', ')}.`, `- Shared: ${SHARED_KEYS.join(', ')}.`,
|
||||
'- Installed names: `gstack-<name>` is key `<name>` (`gstack-ship` is `ship`); `gstack-upgrade` keeps its name; the root skill installs as `gstack`.'];
|
||||
}
|
||||
|
||||
function renderNotHere(inv: Inventory): string[] {
|
||||
return ['## Not in this inventory', '',
|
||||
'- Out of scope: openclaw/skills/* and browser-skills/* (not gstack skills an agent loads from this checkout), and connect-chrome (a symlink to open-gstack-browser, counted once).',
|
||||
...inv.unmapped.map(u => `- ${u.row}: ${u.why}`)];
|
||||
}
|
||||
|
||||
export function renderInventory(inv: Inventory): string {
|
||||
const count = (s: string) => inv.parts.filter(p => p.status === s).length;
|
||||
const planned = inv.parts.filter(p => p.status === 'untested' && !p.ref.startsWith('plan: none')).length;
|
||||
return [
|
||||
'# gstack inventory: every part, its load per run and its evidence', '',
|
||||
`Generated by \`bin/gstack-inventory\` from [evidence.json](inventory/evidence.json), the hand-kept ledger, and lib/context-bill.ts. Measured at commit ${inv.commit} (${inv.date}). This is a dated snapshot: rerun \`bin/gstack-inventory\` to refresh it; \`bin/gstack-inventory --check\` validates the ledger. The rule: [ETHOS.md: 9. Earn Its Place](../ETHOS.md#9-earn-its-place).`, '',
|
||||
`Token counts are offline estimates (${inv.tokenSource}, about ±40%), not billed tokens.`, '',
|
||||
'| Status | Parts |', '|---|---|',
|
||||
...(['helps', 'no-effect', 'hurts'] as const).map(s => `| ${s} | ${count(s)} |`),
|
||||
`| untested, with a plan | ${planned} |`, `| untested, plan none | ${count('untested') - planned} |`, `| safety | ${count('safety')} |`, '',
|
||||
...renderRanking(inv), '', ...renderMeasureNext(inv), '', ...renderShared(inv), '', ...renderDispatches(inv), '',
|
||||
...renderPrior(inv), '', ...renderKeys(inv), '', ...renderNotHere(inv), '',
|
||||
].join('\n');
|
||||
}
|
||||
+7
-1
@@ -11,6 +11,7 @@ import { homedir } from "node:os";
|
||||
import { basename, dirname, join, relative, resolve } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { estimateTokens } from "./context-bill";
|
||||
import { skillEvidence } from "./inventory";
|
||||
|
||||
export const DUE_PCT = 10;
|
||||
|
||||
@@ -28,6 +29,8 @@ export interface SkillBillReport {
|
||||
rows: LevelRow[];
|
||||
baseline: Baseline;
|
||||
evalTasks?: EvalTasks | null;
|
||||
/** gstack checkout only: the skill's row in docs/inventory/evidence.json (ETHOS.md: 9. Earn Its Place). */
|
||||
evidence?: ReturnType<typeof skillEvidence>;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
@@ -196,8 +199,10 @@ export function buildSkillBill(dir: string): SkillBillReport {
|
||||
for (const f of onDemandFiles(dir)) rows.push(row("on-demand", dir, f));
|
||||
const warnings: string[] = [];
|
||||
let evalTasks: EvalTasks | null | undefined;
|
||||
let evidence: SkillBillReport["evidence"];
|
||||
if (kind === "gstack-checkout") {
|
||||
const root = gstackRootOf(dir)!;
|
||||
evidence = skillEvidence(root, dir);
|
||||
const file = join(root, "test", "fixtures", "skill-evals", name, "tasks.json");
|
||||
evalTasks = existsSync(file) ? { ...checkEvalTasks(file), path: `test/fixtures/skill-evals/${name}/tasks.json` } : null;
|
||||
if (evalTasks?.problems.length) warnings.push(`EVAL_TASKS_INVALID: ${evalTasks.problems[0]}${evalTasks.problems.length > 1 ? ` (+${evalTasks.problems.length - 1} more)` : ""}. fix: the task format in docs/designs/SKILL_EVAL_CONTRACT.md`);
|
||||
@@ -206,7 +211,7 @@ export function buildSkillBill(dir: string): SkillBillReport {
|
||||
warnings.push("RENDER_STALE: SKILL.md.tmpl changed but SKILL.md did not. fix: bun run gen:skill-docs");
|
||||
}
|
||||
if (kind === "gstack-install") warnings.push("INSTALLED_GSTACK_SKILL: gstack-upgrade overwrites this folder. fix: edit <skill>/SKILL.md.tmpl in a gstack checkout, or copy the skill to a folder you own");
|
||||
return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, warnings };
|
||||
return { name, dir, kind, authored, rows, baseline: findBaseline(dir, name, authored), evalTasks, evidence, warnings };
|
||||
}
|
||||
|
||||
export function renderSkillBill(r: SkillBillReport): string {
|
||||
@@ -217,6 +222,7 @@ export function renderSkillBill(r: SkillBillReport): string {
|
||||
else lines.push(`BASELINE: ${b.status} (${b.reason})`);
|
||||
if (r.evalTasks) lines.push(`EVAL TASKS: ${r.evalTasks.path} (${r.evalTasks.count} tasks)`);
|
||||
else if (r.evalTasks === null) lines.push(`EVAL TASKS: none; a diet writes test/fixtures/skill-evals/${r.name}/tasks.json before any cut`);
|
||||
if (r.evidence) lines.push(r.evidence.line);
|
||||
lines.push(...r.warnings);
|
||||
lines.push(`Commit line: Skill-Size: <before>→<after> bytes (${r.authored})`);
|
||||
return lines.join("\n");
|
||||
|
||||
+1
@@ -37,6 +37,7 @@
|
||||
{ "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" },
|
||||
{ "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" },
|
||||
{ "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" },
|
||||
{ "file": "lib/inventory.ts", "movedFrom": "(new: the R56 evidence ledger check and the inventory behind bin/gstack-inventory, GST-914)" },
|
||||
{ "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" },
|
||||
{ "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" },
|
||||
{ "file": "lib/owner-brief.ts", "movedFrom": "(new: the one-page owner brief and draft-direction close behind bin/gstack-owner-brief, plan E3)" },
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* R56 (ETHOS.md: 9. Earn Its Place): every part of gstack has a row in
|
||||
* docs/inventory/evidence.json, validated by its status's own fields, and
|
||||
* bin/gstack-inventory ranks the parts by load per run into docs/inventory.md.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { discoverTemplates } from '../scripts/discover-skills';
|
||||
import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-inventory');
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-inventory-'));
|
||||
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
||||
|
||||
const codes = (key: string, entry: unknown) => validateEntry(key, entry).map(p => p.code);
|
||||
const run = (args: string[]) => spawnSync(BIN, args, { cwd: ROOT, encoding: 'utf8', timeout: 120_000 });
|
||||
|
||||
const VERDICT = { sources: ['gstack-evals run r9 results.json'], models: ['claude-opus-5-5', 'gpt-6.1-sol'], prompt: 'preset', scope: 'the full procedure, told', masked: true };
|
||||
const NEW_UNTESTED = { status: 'untested', reason: 'catches missed migrations', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-168', measure_by: '2026-12-01' };
|
||||
|
||||
describe('the key list', () => {
|
||||
test('keys are the skill directories discoverTemplates finds, the root skill as gstack, then the fixed shared keys', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
const skills = discoverTemplates(ROOT).map(t => path.dirname(t.tmpl)).map(d => (d === '.' ? ROOT_KEY : d));
|
||||
expect(keys).toEqual([...skills.sort(), ...SHARED_KEYS]);
|
||||
expect(keys).toContain('gstack');
|
||||
expect(keys).toContain('gstack-upgrade');
|
||||
expect(keys).not.toContain('connect-chrome');
|
||||
expect(new Set(keys).size).toBe(keys.length);
|
||||
});
|
||||
|
||||
test('installed names map to keys: gstack-<name> to <name>, gstack-upgrade stays literal', () => {
|
||||
expect(keyOf('gstack-ship')).toBe('ship');
|
||||
expect(keyOf('ship')).toBe('ship');
|
||||
expect(keyOf('gstack-upgrade')).toBe('gstack-upgrade');
|
||||
expect(keyOf('gstack')).toBe('gstack');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the ledger covers every key and validates', () => {
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(ROOT, LEDGER_PATH), 'utf8'));
|
||||
|
||||
test('no missing and no extra keys, and every row passes its status schema', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
expect(Object.keys(ledger.parts).filter(k => !keys.includes(k))).toEqual([]);
|
||||
expect(keys.filter(k => !(k in ledger.parts))).toEqual([]);
|
||||
expect(validateLedger(ledger, keys)).toEqual([]);
|
||||
});
|
||||
|
||||
test('the initial backlog is grandfathered and the guards are safety rows', () => {
|
||||
for (const [key, row] of Object.entries<any>(ledger.parts)) expect([key, row.grandfathered]).toEqual([key, true]);
|
||||
for (const key of ['careful', 'freeze', 'guard', 'unfreeze', 'cso']) expect(ledger.parts[key].status).toBe('safety');
|
||||
for (const key of ['capy-skill', 'capy-factory']) expect(ledger.parts[key]).toMatchObject({ status: 'untested', plan: 'none' });
|
||||
});
|
||||
|
||||
test('a missing and an extra key are each named', () => {
|
||||
const keys = inventoryKeys(ROOT);
|
||||
const { ship, ...rest } = ledger.parts;
|
||||
const problems = validateLedger({ ...ledger, parts: { ...rest, 'no-such-part': ship } }, keys);
|
||||
expect(problems.map(p => `${p.key} ${p.code}`)).toEqual(['ship LEDGER_KEY_MISSING', 'no-such-part LEDGER_KEY_EXTRA']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the per-status schema', () => {
|
||||
test('passing rows: a grandfathered untested part with plan none, a new part with a dated plan, a fair verdict, a safety guard', () => {
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none' })).toEqual([]);
|
||||
expect(codes('a', NEW_UNTESTED)).toEqual([]);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'docs/evals/demo/PREREG.md' })).toEqual([]);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT })).toEqual([]);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1'] })).toEqual([]);
|
||||
expect(codes('a', { status: 'no-effect', ...VERDICT, r48: { verdict: 'CUT-CANDIDATE', ref: 'why-null.md' } })).toEqual([]);
|
||||
expect(codes('a', { status: 'safety', reason: 'blocks rm -rf' })).toEqual([]);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['stub', 'unmasked'] }] })).toEqual([]);
|
||||
});
|
||||
|
||||
test('a new untested part needs reason, a concrete plan, owner and measure_by; plan none is refused', () => {
|
||||
const { measure_by, ...noDate } = NEW_UNTESTED;
|
||||
expect(codes('a', noDate)).toEqual(['LEDGER_MEASURE_BY_MISSING']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, measure_by: 'soon' })).toEqual(['LEDGER_MEASURE_BY_MISSING']);
|
||||
const { owner, ...noOwner } = NEW_UNTESTED;
|
||||
expect(codes('a', noOwner)).toEqual(['LEDGER_OWNER_MISSING']);
|
||||
const { reason, ...noReason } = NEW_UNTESTED;
|
||||
expect(codes('a', noReason)).toEqual(['LEDGER_REASON_MISSING']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'none' })).toEqual(['LEDGER_PLAN_NONE_NEW']);
|
||||
expect(codes('a', { ...NEW_UNTESTED, plan: 'we will test it later' })).toEqual(['LEDGER_PLAN_NOT_CONCRETE']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true })).toEqual(['LEDGER_PLAN_MISSING']);
|
||||
});
|
||||
|
||||
test('helps, no-effect and hurts need sources, frontier models, the preset prompt, a scope and a masked no-gstack arm', () => {
|
||||
const { sources, ...noSources } = VERDICT;
|
||||
expect(codes('a', { status: 'helps', ...noSources })).toEqual(['LEDGER_SOURCES_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, sources: [] })).toEqual(['LEDGER_SOURCES_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-haiku-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'claude-sonnet-5-5'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, models: ['claude-opus-5-5', 'particle-1', 'deepseek-v4.1-flash'] })).toEqual(['LEDGER_MODELS_NOT_FRONTIER']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, prompt: 'sdk-stub' })).toEqual(['LEDGER_PROMPT_NOT_PRESET']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, scope: '', r48: { verdict: 'KEEP', ref: 'x' } })).toEqual(['LEDGER_SCOPE_MISSING']);
|
||||
expect(codes('a', { status: 'helps', ...VERDICT, masked: false })).toEqual(['LEDGER_UNMASKED']);
|
||||
});
|
||||
|
||||
test('no-effect and hurts without an R48 verdict and its reference are refused', () => {
|
||||
expect(codes('a', { status: 'no-effect', ...VERDICT })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'DROP', ref: 'x' } })).toEqual(['LEDGER_R48_MISSING']);
|
||||
expect(codes('a', { status: 'hurts', ...VERDICT, r48: { verdict: 'KEEP' } })).toEqual(['LEDGER_R48_MISSING']);
|
||||
});
|
||||
|
||||
test('safety needs a reason; unknown statuses, unknown fields and bad prior entries are refused', () => {
|
||||
expect(codes('a', { status: 'safety' })).toEqual(['LEDGER_REASON_MISSING']);
|
||||
expect(codes('a', { status: 'exempt', reason: 'x' })).toEqual(['LEDGER_STATUS_INVALID']);
|
||||
expect(codes('a', { status: 'safety', reason: 'x', source: 'typo' })).toEqual(['LEDGER_FIELD_UNKNOWN']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ ref: 'run 7', why: ['vibes'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
|
||||
expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', prior: [{ why: ['stub'] }] })).toEqual(['LEDGER_PRIOR_INVALID']);
|
||||
});
|
||||
});
|
||||
|
||||
/** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */
|
||||
function fixtureTree(dir: string, parts: (keys: string[]) => Record<string, unknown>): string {
|
||||
const write = (rel: string, text: string) => { fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); fs.writeFileSync(path.join(dir, rel), text); };
|
||||
const skill = (rel: string, name: string, tier: number, body: string) => {
|
||||
const fm = `---\nname: ${name}\npreamble-tier: ${tier}\ndescription: |\n Does ${name} things.\n---\n`;
|
||||
write(path.join(rel, 'SKILL.md.tmpl'), fm + body);
|
||||
write(path.join(rel, 'SKILL.md'), fm + '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n' + body);
|
||||
};
|
||||
skill('', 'gstack', 1, 'router\n');
|
||||
skill('alpha', 'alpha', 2, 'a'.repeat(4000) + '\n');
|
||||
skill('beta', 'beta', 2, 'b'.repeat(1000) + '\n');
|
||||
write('alpha/sections/preamble.md', 'p'.repeat(800));
|
||||
write('beta/sections/preamble.md', 'p'.repeat(800));
|
||||
for (const part of SHARED_PARTS) for (const f of part.files) write(f, 'x'.repeat(500) + '\n');
|
||||
for (const d of DISPATCHES) if (!fs.existsSync(path.join(dir, d.file))) write(d.file, 'filler\n');
|
||||
for (const d of DISPATCHES) fs.appendFileSync(path.join(dir, d.file), `${d.quote}\n`);
|
||||
const keys = inventoryKeys(dir);
|
||||
write(LEDGER_PATH, JSON.stringify({ schema_version: 1, parts: parts(keys), unmapped: [{ row: 'gstack as a whole', why: 'not a single text part' }] }, null, 2));
|
||||
return dir;
|
||||
}
|
||||
|
||||
const grandfathered = (keys: string[]) => Object.fromEntries(keys.map(k => [k, k === 'beta' ? { status: 'safety', grandfathered: true, reason: 'a guard' } : { status: 'untested', grandfathered: true, plan: 'none' }]));
|
||||
|
||||
describe('bin/gstack-inventory', () => {
|
||||
test('determinism: two runs on a fixed tree write the same docs/inventory.md and the same --json', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'fixed'), grandfathered);
|
||||
const first = run(['--root', dir]);
|
||||
expect(first.status, first.stderr).toBe(0);
|
||||
const a = fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8');
|
||||
const second = run(['--root', dir]);
|
||||
expect(second.status, second.stderr).toBe(0);
|
||||
expect(fs.readFileSync(path.join(dir, 'docs/inventory.md'), 'utf8')).toBe(a);
|
||||
expect(run(['--root', dir, '--json']).stdout).toBe(run(['--root', dir, '--json']).stdout);
|
||||
expect(a).toContain('Measured at commit none');
|
||||
expect(a.indexOf('| alpha |')).toBeLessThan(a.indexOf('| beta |'));
|
||||
const start = a.indexOf('## Measure next');
|
||||
const next = a.slice(start, a.indexOf('\n## ', start + 1));
|
||||
expect(next).toContain('alpha');
|
||||
expect(next).not.toContain('| beta |');
|
||||
expect(a).toContain('gstack as a whole');
|
||||
expect(a).toContain('gstack-<name>');
|
||||
for (const d of DISPATCHES) expect(a).toContain(`${d.file}:`);
|
||||
});
|
||||
|
||||
test('--check: ok on a complete ledger; a missing key exits 1 with its code', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'check'), grandfathered);
|
||||
const ok = run(['--root', dir, '--check']);
|
||||
expect(ok.status, ok.stderr).toBe(0);
|
||||
expect(ok.stdout).toMatch(/^INVENTORY: ok keys=\d+/m);
|
||||
const bad = fixtureTree(path.join(tmp, 'check-bad'), keys => { const p = grandfathered(keys); delete p.alpha; return p; });
|
||||
const r = run(['--root', bad, '--check']);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain('alpha: no row in docs/inventory/evidence.json (LEDGER_KEY_MISSING)');
|
||||
expect(run(['--root', bad]).status).toBe(1);
|
||||
});
|
||||
|
||||
test('--check: a cited dispatch source that lost its quote is named', () => {
|
||||
const dir = fixtureTree(path.join(tmp, 'drift'), grandfathered);
|
||||
fs.writeFileSync(path.join(dir, DISPATCHES[0]!.file), 'rewritten\n');
|
||||
const r = run(['--root', dir, '--check']);
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain('(INVENTORY_SOURCE_MISSING)');
|
||||
});
|
||||
|
||||
test('the real ledger passes --check', () => {
|
||||
const r = run(['--check']);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
});
|
||||
});
|
||||
+33
-2
@@ -47,7 +47,7 @@ describe('gstack-skill-bill', () => {
|
||||
const root = gstackRepo();
|
||||
|
||||
test('a gstack checkout skill reports every level and its last diet', () => {
|
||||
const r = bill(root, ['demo']);
|
||||
const r = bill(root, ['./demo']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
|
||||
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
|
||||
@@ -58,7 +58,7 @@ describe('gstack-skill-bill', () => {
|
||||
|
||||
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
|
||||
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
|
||||
const r = bill(root, ['demo', '--json']);
|
||||
const r = bill(root, ['./demo', '--json']);
|
||||
const report = JSON.parse(r.stdout);
|
||||
expect(report.baseline.status).toBe('found');
|
||||
expect(report.baseline.due).toBe(true);
|
||||
@@ -114,6 +114,37 @@ describe('gstack-skill-bill', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-skill-bill EVIDENCE line (docs/inventory/evidence.json)', () => {
|
||||
const root = path.join(tmp, 'evidence-repo');
|
||||
const ledger = (parts: Record<string, unknown>) => {
|
||||
fs.mkdirSync(path.join(root, 'docs', 'inventory'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'docs', 'inventory', 'evidence.json'), JSON.stringify({ schema_version: 1, parts }));
|
||||
};
|
||||
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
|
||||
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
|
||||
skill(path.join(root, 'demo'), 'demo', 'body\n', 'SKILL.md.tmpl');
|
||||
|
||||
test('each status prints its source, plan or reason; a skill with no row says so', () => {
|
||||
ledger({ demo: { status: 'untested', grandfathered: true, plan: 'none' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: none (grandfathered)');
|
||||
ledger({ demo: { status: 'untested', reason: 'r', plan: 'test/fixtures/skill-evals/demo/tasks.json', owner: 'GSTA-1', measure_by: '2026-12-01' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: untested plan: test/fixtures/skill-evals/demo/tasks.json (owner GSTA-1, measure by 2026-12-01)');
|
||||
ledger({ demo: { status: 'helps', sources: ['run-9/results.json'], models: ['claude-opus-5-5'], prompt: 'preset', scope: 's', masked: true } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: helps sources: run-9/results.json');
|
||||
ledger({ demo: { status: 'safety', reason: 'blocks rm -rf' } });
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: safety reason: blocks rm -rf');
|
||||
ledger({});
|
||||
expect(bill(root, ['./demo']).stdout).toContain('EVIDENCE: missing: no demo row in docs/inventory/evidence.json');
|
||||
expect(JSON.parse(bill(root, ['./demo', '--json']).stdout).evidence).toMatchObject({ key: 'demo', status: 'missing' });
|
||||
});
|
||||
|
||||
test('the real checkout: a guard prints its safety reason', () => {
|
||||
const r = spawnSync(BIN, ['careful'], { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.stdout).toMatch(/^EVIDENCE: safety reason: .+/m);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-skill-bill baselines and the root skill', () => {
|
||||
test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => {
|
||||
const root = path.join(tmp, 'gstack-root');
|
||||
|
||||
Reference in new issue
Block a user