diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa74a6704..b16b6fedb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -814,7 +814,7 @@ Write or change skill prose with `/write-skill` (edit ladder, levels, `gstack-sk 3. The dynamic template discovery picks it up for every host render and for `gstack/llms.txt`. Two things it does not fill in: a row in `SKILL_REQUIREMENTS` in `lib/doctor-components.ts` (the doctor's `--for ` table; `test/doctor-components.test.ts` fails on a skill directory without one, and the same gen run rerenders `bin/gstack-doctor-components.sh`), and the catalog rows readers find the skill through (README.md, AGENTS.md, `docs/skills.md`, `docs/PROJECT_STRUCTURE.md`, the root `SKILL.md.tmpl` router) 4. Budget it: run `bun test/helpers/capture-context-budget.ts` and commit the refreshed `test/fixtures/context-budget.json` — the context-budget ratchet fails any skill without a ceiling; the catalog budget (`test/catalog-budget.test.ts`) fails until its cap is ratcheted by the protocol in its failure message, and the number is quoted in CLAUDE.md, this file and `docs/CONTRIBUTOR_REFERENCE.md` 5. Commit `{skill}/SKILL.md` (and the generated `{skill}/sections/preamble.md` when the template's `preamble-tier` is 2 or higher), external host output is generated at setup time and gitignored -6. Give it a row in the evidence ledger, `docs/inventory/evidence.json` ([ETHOS.md: 9. Earn Its Place](ETHOS.md#9-earn-its-place)): evidence that it helps on frontier models (`sources`, `models`, `prompt: preset`, `scope`, `masked: true`), or `status: untested` with its `reason`, a concrete `plan` (a `test/fixtures/skill-evals//tasks.json` or an /eval-plan preregistration), an `owner` and a `measure_by` date. `test/inventory.test.ts` fails on a skill without a valid row; `bin/gstack-inventory --check` names the problem, and `bin/gstack-inventory` refreshes the dated snapshot in `docs/inventory.md` +6. Give it a row in the evidence ledger, `docs/inventory/evidence.json` ([ETHOS.md: 9. Earn Its Place](ETHOS.md#9-earn-its-place)): evidence that it helps on frontier models (`sources`, `models`, `prompt: preset`, `scope`, `masked: true`, and `caveats` for any limits on the result), or `status: untested` with its `reason`, a concrete `plan` (a `test/fixtures/skill-evals//tasks.json` or an /eval-plan preregistration), an `owner` and a `measure_by` date. `test/inventory.test.ts` fails on a skill without a valid row; `bin/gstack-inventory --check` names the problem, and `bin/gstack-inventory` refreshes the dated snapshot in `docs/inventory.md` ## Conductor workspaces diff --git a/changelog.d/capy-gst-1370-review-helps-ledger.md b/changelog.d/capy-gst-1370-review-helps-ledger.md new file mode 100644 index 000000000..7cff5797f --- /dev/null +++ b/changelog.d/capy-gst-1370-review-helps-ledger.md @@ -0,0 +1 @@ +- **The evidence ledger records /review as helps, with its two caveats beside the status.** R2 at the after pin (gstack-evals #207, superseded by #211) measured /review with the contract Stop hook on Opus 5.5 at gstack 699a41cb: blocking recall 26.2% against 3.6% for plain Opus 5.5 with no gstack, +22.6 points (95% CI +11.0 to +33.7). The `review` row in `docs/inventory/evidence.json` now says helps, with its sources, scope and R48 KEEP. A new `caveats` field, allowed only on helps, no-effect and hurts rows, holds the result's limits, and `docs/inventory.md` and `gstack-skill-bill` print them beside the status: /review and the hook count as one unit (42 of 66 sessions ran the readers only after the hook blocked), and 41 of the suite's 42 bugs came from gstack-family reviews. The earlier review results stay under `prior`. diff --git a/docs/inventory/evidence.json b/docs/inventory/evidence.json index d85ce971a..b3fc00d77 100644 --- a/docs/inventory/evidence.json +++ b/docs/inventory/evidence.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.", + "note": "R56 evidence ledger (ETHOS.md: 9. Earn Its Place). One row per inventory key (docs/inventory.md lists them); bin/gstack-inventory --check validates it and test/inventory.test.ts runs that check. Statuses and their required fields: helps, no-effect, hurts need sources, models (frontier per R54), prompt: preset, scope and masked: true, and no-effect and hurts also r48 {verdict, ref}; caveats lists a verdict's limits, shown beside its status; untested needs plan (a reference, or none only when grandfathered), and a new part also reason, owner and measure_by; safety needs reason. prior lists earlier results that do not count yet, with why. Seeded 2026-11-02 from GEVA-2's table (org-gstack-gbrain/research/gstack-part-evidence.json, gstack dac71371a); every row present at that base is grandfathered.", "seed_source": "org-gstack-gbrain/research/gstack-part-evidence.json (GEVA-156 for GST-915, 2026-11-02, gstack-evals main 409ff47)", "parts": { "autoplan": { @@ -420,9 +420,26 @@ "seed": "GEVA-2, 2026-11-02: retro: untested (never measured)" }, "review": { - "status": "untested", + "status": "helps", "grandfathered": true, - "plan": "R2 (GEVA-2, 2026-11-02): gstack-evals wiki/plans/2026-11-02-geva-122-r2-amendment (#129, main eae6efc), told /review vs a no-gstack arm, Opus 5.5, preset on both arms", + "sources": [ + "gstack-evals #207 (baa599a9) wiki/board/2026-11-04-geva-271-r2-after-pin-results", + "gstack-evals #211 (81daa79) wiki/board/2026-11-04-geva-281-r2-after-pin-results-home-field" + ], + "models": [ + "claude-opus-5-5" + ], + "prompt": "preset", + "scope": "/review as a Claude Code user installs it at gstack 699a41cb (./setup --host claude), contract Stop hook included, told /review, against plain Opus 5.5 with no gstack; Claude Code CLI 2.1.314 on the claude_code preset; stacks-r1, 33 units, 42 blocking bugs, 2 passes: blocking recall 22/84 (26.2%) vs 3/84 (3.6%), +22.6 points [95% PR-clustered CI +11.0, +33.7], MDD 24.5", + "masked": true, + "r48": { + "verdict": "KEEP", + "ref": "gstack-evals #211, wiki/board/2026-11-04-geva-281-r2-after-pin-results-home-field (What we found: R48 KEEP, rr-decide.py KEEP)" + }, + "caveats": [ + "one unit: /review and the contract Stop hook were measured together, and neither gets credit alone; 42 of 66 gstack sessions ran the review readers only after the hook blocked them (24 unprompted), and that split is not randomized", + "home-field bias in gstack's favor: 41 of the 42 blocking bugs in stacks-r1 came from gstack-family reviews, so +22.6 may run high; a suite seeded from other reviewers' bugs would remove the bias (GEVA-2 lists it as future work)" + ], "prior": [ { "ref": "gstack-evals #108 stacks round 1 review (GEVA-89), wiki/board/2026-11-01-geva-89-stacks-round-1-review-results", @@ -450,11 +467,8 @@ "note": "/review's prompts through the Messages API; Opus recall 44% vs 39% plain, 12 bugs" } ], - "r48": { - "verdict": "FIX-METHOD", - "ref": "org-gstack-gbrain/research/why-null-stacks-r1.md (GEVA-100); R2 is the fix" - }, - "seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)" + "seed": "GEVA-2, 2026-11-02: review: measurement under way (R2)", + "note": "R2 at the after pin: GEVA-271 under prereg #194 (amending #154), runs review-units-20261104T070239Z-cbff (gstack-full) and review-units-20261104T123631Z-3fb3 (none-rr); #211 (GEVA-281) supersedes #207's post with every number unchanged and adds the home-field limit. Until then, this row was untested with R48 FIX-METHOD (org-gstack-gbrain/research/why-null-stacks-r1.md, GEVA-100), plan R2 (gstack-evals #129). R2's first two attempts gave no outcome: GEVA-175 (procedure gate 10 of 19) and GEVA-229 (exposure smoke 2 of 3). masked: the no-gstack arm ran in the rig's Docker sandbox with no gstack install, and #207's read audit of its 66 transcripts found no read of ~/.capy/drive or ~/.local/share/gstack. Only Opus 5.5 was measured; GPT-6.1 Sol was not." }, "scrape": { "status": "untested", diff --git a/lib/inventory.ts b/lib/inventory.ts index 9615e4841..ef05c437f 100644 --- a/lib/inventory.ts +++ b/lib/inventory.ts @@ -66,7 +66,7 @@ export const PRIOR_WHY: Record = { 'not-claude-code': 'ran outside Claude Code (Messages API, Codex or Capy)', 'variant-only': 'variant against variant, no no-gstack arm', }; -const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'seed', 'aliases', 'note']; +const FIELDS = ['status', 'grandfathered', 'plan', 'prior', 'reason', 'owner', 'measure_by', 'sources', 'models', 'prompt', 'scope', 'masked', 'r48', 'caveats', 'seed', 'aliases', 'note']; export type LedgerCode = | 'LEDGER_PARSE' | 'LEDGER_KEY_MISSING' | 'LEDGER_KEY_EXTRA' | 'LEDGER_STATUS_INVALID' | 'LEDGER_FIELD_UNKNOWN' @@ -75,7 +75,7 @@ export type LedgerCode = | 'LEDGER_REASON_MISSING' | 'LEDGER_OWNER_MISSING' | 'LEDGER_MEASURE_BY_MISSING' | 'LEDGER_PRIOR_INVALID' | 'LEDGER_FIELD_INVALID' | 'INVENTORY_SOURCE_MISSING' | 'INVENTORY_PART_MISSING'; export interface LedgerProblem { key: string; code: LedgerCode; message: string } -export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; seed?: string; aliases?: string[]; note?: string } +export interface LedgerEntry { status: Status; grandfathered?: boolean; plan?: string; prior?: Array<{ ref: string; why: string[]; note?: string }>; reason?: string; owner?: string; measure_by?: string; sources?: string[]; models?: string[]; prompt?: string; scope?: string; masked?: boolean; r48?: { verdict: string; ref: string }; caveats?: string[]; seed?: string; aliases?: string[]; note?: string } export interface Ledger { parts: Record; unmapped?: Array<{ row: string; why: string }>; [k: string]: unknown } const isObj = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); @@ -100,6 +100,8 @@ export function validateEntry(key: string, entry: unknown): LedgerProblem[] { add('LEDGER_PRIOR_INVALID', `prior needs [{ref, why: [...]}] with why from: ${Object.keys(PRIOR_WHY).join(', ')}`); if (entry.aliases !== undefined && !(Array.isArray(entry.aliases) && entry.aliases.length > 0 && entry.aliases.every(str))) add('LEDGER_FIELD_INVALID', 'aliases needs a list of names'); const status = entry.status; + if (entry.caveats !== undefined && !((status === 'helps' || status === 'no-effect' || status === 'hurts') && Array.isArray(entry.caveats) && entry.caveats.length > 0 && entry.caveats.every(str))) + add('LEDGER_FIELD_INVALID', 'caveats is a list of limits on a helps, no-effect or hurts result'); if (status === 'helps' || status === 'no-effect' || status === 'hurts') { if (!Array.isArray(entry.sources) || !entry.sources.length || !entry.sources.every(str)) add('LEDGER_SOURCES_MISSING', `${status} needs sources: one or more run ids or results files`); if (!frontierModels(entry.models)) add('LEDGER_MODELS_NOT_FRONTIER', `models must include ${FRONTIER_MODELS.join(' or ')} plus at most one named low-cost primary, never Haiku or Sonnet (R54)`); @@ -164,7 +166,7 @@ export const keyOf = (name: string) => (name !== 'gstack-upgrade' && name.starts export function evidenceRef(e: LedgerEntry): string { if (e.status === 'untested') return `plan: ${e.plan}${e.grandfathered ? ' (grandfathered)' : ` (owner ${e.owner}, measure by ${e.measure_by})`}`; if (e.status === 'safety') return `reason: ${e.reason}`; - return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}`; + return `sources: ${(e.sources ?? []).join(', ')}${e.r48 ? `; R48 ${e.r48.verdict}` : ''}${e.caveats ? `; caveats: ${e.caveats.join('; ')}` : ''}`; } /** gstack-skill-bill's EVIDENCE line for one skill folder in a gstack checkout. */ diff --git a/test/inventory.test.ts b/test/inventory.test.ts index a26e5e64d..958d75e8d 100644 --- a/test/inventory.test.ts +++ b/test/inventory.test.ts @@ -9,7 +9,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { discoverTemplates } from '../scripts/discover-skills'; -import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, installReason, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory'; +import { DISPATCHES, LEDGER_PATH, ROOT_KEY, SHARED_KEYS, SHARED_PARTS, evidenceRef, installReason, inventoryKeys, keyOf, validateEntry, validateLedger } from '../lib/inventory'; const ROOT = path.resolve(import.meta.dir, '..'); const BIN = path.join(ROOT, 'bin', 'gstack-inventory'); @@ -129,6 +129,16 @@ describe('the per-status schema', () => { expect(codes('a', { status: 'safety', reason: 'x', aliases: [] })).toEqual(['LEDGER_FIELD_INVALID']); expect(codes('a', { status: 'safety', reason: 'x', aliases: ['other-name'] })).toEqual([]); }); + + test('caveats is a list of limits on a verdict row, shown beside its sources', () => { + const caveats = ['measured with the hook', 'suite seeded from its own reviews']; + expect(codes('a', { status: 'helps', ...VERDICT, caveats })).toEqual([]); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: [] })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: 'one string' })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'helps', ...VERDICT, caveats: [''] })).toEqual(['LEDGER_FIELD_INVALID']); + expect(codes('a', { status: 'untested', grandfathered: true, plan: 'none', caveats })).toEqual(['LEDGER_FIELD_INVALID']); + expect(evidenceRef({ status: 'helps', ...VERDICT, caveats } as any)).toBe('sources: gstack-evals run r9 results.json; caveats: measured with the hook; suite seeded from its own reviews'); + }); }); /** A small gstack-shaped tree: a root skill, two skills, the shared-part files and the cited dispatch sources. */