mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
test(E1): pin CLAUDE.md facts, /ship step references and state-root literals
New free test test/instruction-facts.test.ts (CEO/DX/Eng E1 items; DX-13 one
named test() per check, each failure prints both values, file:line and the
fix):
- 'CLAUDE.md names every TREE_MUTATING file and does not call the set
empty': imports TREE_MUTATING from scripts/test-free-shards.ts (today
test/bootstrap-retention.test.ts).
- 'CLAUDE.md states the current catalog ceiling from
test/catalog-budget.test.ts': the "`CATALOG_BUDGET_TOKEN_EQUIVALENTS`
(N today" value must equal the constant (1,194).
- 'CLAUDE.md points at package.json for detach timeouts instead of stating
numbers': no eval:bg* --timeout value appears in CLAUDE.md (plain or with
commas), and the detach section names package.json's eval:bg:gate /
eval:bg:periodic and states no timeout number.
- 'every explicit /ship Step reference names a defined /ship step with the
expected title': matches only `/ship Step N` and `/ship's Step N`
(backticked or not) across every .tmpl and scripts/resolvers/; each must
be a table row whose step is a heading in ship/SKILL.md.tmpl or
ship/sections/*.md.tmpl whose title contains the row's subject, and the
reference's line or enclosing heading must mention that subject; stale rows
fail. Today: document-release release-body Step 7 -> /ship Step 14
(TODOS.md), plus the incident-history comment in
scripts/resolvers/constants.ts ("/ship Step 18"), kept as a historical row
with a reason (handoff: that comment's step is stale; the dispatch is Step
14.5).
- 'negative control: moving a reference to another existing /ship step
fails': rewrites the TODOS reference to Step 12 in memory and asserts the
checker reports it.
- 'every ~/.gstack/projects/ and ~/.gstack/analytics/ hit is allowlisted with
a reason' and 'every state-root allowlist row still matches a hit': all 37
current hits in .tmpl and scripts/resolvers/ are covered by 36 rows, each with one of four
reasons (user-facing description; gbrain context_queries manifest glob read
literally by bin/gstack-brain-context-load.ts; source comment; pending
conversion tracked in TODOS.md). The readiness-gate EVAL_DIR rows reuse the
reason from test/state-root-ratchet.allowlist.json. New hits fail, and
converting a literal fails until its row is deleted.
Mutation pass (scratch worktree): 15/15 as expected; drifting the catalog
number in CLAUDE.md or the constant, dropping or emptying TREE_MUTATING text,
adding a TREE_MUTATING key, copying a detach timeout into CLAUDE.md,
moving/undefining/retitling a /ship reference, a new /ship reference, a new
state-root literal and a converted allowlisted literal all go red; unrelated
rewording stays green. Log: .context/mutation-instruction-facts.md.
This commit is contained in:
garrytan committed
1 parent
4819d8e2b8
commit
727481797e
1 file changed
+240
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Instruction facts pinned to their sources (free, static).
|
||||
*
|
||||
* CLAUDE.md and skill templates state facts that code owns: which free test
|
||||
* files run serially, the catalog token ceiling, where the detach timeouts
|
||||
* live, which /ship step another skill points at, and where gstack state is
|
||||
* read and written. Each check below fails with both values, the file:line,
|
||||
* and the fix, so drift is repaired at the source instead of rediscovered by
|
||||
* an agent following stale prose.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { TREE_MUTATING } from '../scripts/test-free-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const lineOf = (text: string, index: number) => text.slice(0, index).split('\n').length;
|
||||
const claudeMd = read('CLAUDE.md');
|
||||
|
||||
function walk(dir: string, keep: (rel: string) => boolean): string[] {
|
||||
const out: string[] = [];
|
||||
for (const entry of fs.readdirSync(path.join(ROOT, dir), { withFileTypes: true })) {
|
||||
const rel = dir === '.' ? entry.name : `${dir}/${entry.name}`;
|
||||
if (entry.isDirectory()) {
|
||||
if (['node_modules', '.git', 'test', 'dist', '.context'].includes(entry.name) || entry.name.startsWith('.')) continue;
|
||||
out.push(...walk(rel, keep));
|
||||
} else if (keep(rel)) out.push(rel);
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
const templates = walk('.', rel => rel.endsWith('.tmpl'));
|
||||
const resolvers = walk('scripts/resolvers', rel => rel.endsWith('.ts'));
|
||||
|
||||
describe('CLAUDE.md facts match the code that owns them', () => {
|
||||
test('CLAUDE.md names every TREE_MUTATING file and does not call the set empty', () => {
|
||||
const at = claudeMd.indexOf('`TREE_MUTATING`');
|
||||
expect(at, 'CLAUDE.md:? no longer mentions `TREE_MUTATING`; fix: describe scripts/test-free-shards.ts TREE_MUTATING in the Testing section').toBeGreaterThanOrEqual(0);
|
||||
const paragraph = claudeMd.slice(at, claudeMd.indexOf('\n\n', at));
|
||||
const keys = Object.keys(TREE_MUTATING);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
for (const key of keys) {
|
||||
expect(paragraph.includes(key),
|
||||
`CLAUDE.md:${lineOf(claudeMd, at)} TREE_MUTATING text omits ${key}; code (scripts/test-free-shards.ts) lists ${JSON.stringify(keys)}; fix: name every key in that paragraph`).toBe(true);
|
||||
}
|
||||
expect(paragraph, `CLAUDE.md:${lineOf(claudeMd, at)} calls TREE_MUTATING empty while code lists ${JSON.stringify(keys)}; fix: name the files`)
|
||||
.not.toMatch(/TREE_MUTATING`[^.]*\b(?:is|now) empty\b|no files? (?:still )?run/i);
|
||||
});
|
||||
|
||||
test('CLAUDE.md states the current catalog ceiling from test/catalog-budget.test.ts', () => {
|
||||
const source = read('test/catalog-budget.test.ts');
|
||||
const constant = /const CATALOG_BUDGET_TOKEN_EQUIVALENTS = ([\d_]+);/.exec(source);
|
||||
expect(constant, 'test/catalog-budget.test.ts no longer declares CATALOG_BUDGET_TOKEN_EQUIVALENTS as a literal; fix: update this parser').not.toBeNull();
|
||||
const value = Number(constant![1]!.replace(/_/g, ''));
|
||||
const at = claudeMd.indexOf('`CATALOG_BUDGET_TOKEN_EQUIVALENTS`');
|
||||
expect(at, 'CLAUDE.md no longer names `CATALOG_BUDGET_TOKEN_EQUIVALENTS`; fix: name the constant next to its value').toBeGreaterThanOrEqual(0);
|
||||
const stated = /^`CATALOG_BUDGET_TOKEN_EQUIVALENTS` \(([\d,]+) today/.exec(claudeMd.slice(at));
|
||||
expect(stated, `CLAUDE.md:${lineOf(claudeMd, at)} names the constant without "(<value> today"; fix: state ${value.toLocaleString('en-US')}`).not.toBeNull();
|
||||
const claimed = Number(stated![1]!.replace(/,/g, ''));
|
||||
expect(claimed, `CLAUDE.md:${lineOf(claudeMd, at)} says ${claimed}, test/catalog-budget.test.ts says ${value}; fix: update CLAUDE.md to ${value.toLocaleString('en-US')}`).toBe(value);
|
||||
});
|
||||
|
||||
test('CLAUDE.md points at package.json for detach timeouts instead of stating numbers', () => {
|
||||
const scripts = JSON.parse(read('package.json')).scripts as Record<string, string>;
|
||||
const timeouts = Object.entries(scripts)
|
||||
.filter(([name, command]) => name.startsWith('eval:bg') && /--timeout \d+/.test(command))
|
||||
.map(([name, command]) => ({ name, seconds: /--timeout (\d+)/.exec(command)![1]! }));
|
||||
expect(timeouts.map(t => t.name)).toEqual(expect.arrayContaining(['eval:bg:gate', 'eval:bg:periodic']));
|
||||
for (const { name, seconds } of timeouts) {
|
||||
for (const spelled of [seconds, Number(seconds).toLocaleString('en-US')]) {
|
||||
const at = claudeMd.indexOf(spelled);
|
||||
expect(at, `CLAUDE.md:${lineOf(claudeMd, at)} states ${spelled}, the ${name} --timeout in package.json; fix: point at package.json's \`${name}\` instead of copying the number`).toBe(-1);
|
||||
}
|
||||
}
|
||||
const start = claudeMd.indexOf('## Running evals as an agent');
|
||||
expect(start).toBeGreaterThanOrEqual(0);
|
||||
const section = claudeMd.slice(start, claudeMd.indexOf('\n## ', start + 1));
|
||||
expect(section).toContain("package.json's `eval:bg:gate` / `eval:bg:periodic`");
|
||||
const stale = /--timeout \d+|\b\d[\d,]{3,}\s*(?:s|sec|seconds)\b/.exec(section);
|
||||
expect(stale?.[0] ?? null,
|
||||
`CLAUDE.md:${stale ? lineOf(claudeMd, start + stale.index) : '?'} states a timeout number in the detach section; package.json owns it; fix: remove the number`).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// Explicit cross-skill references to a /ship step. Each row names the step's
|
||||
// title subject; the reference's own line or enclosing heading must mention it,
|
||||
// so pointing at a different existing step fails as well as a missing one.
|
||||
// A historical row records an incident-history comment, not an instruction;
|
||||
// it is exempt from title checks but must still match its reference.
|
||||
const SHIP_STEP_REFERENCES: Array<{ file: string; step: string; title?: string; historical?: string }> = [
|
||||
{ file: 'document-release/sections/release-body.md.tmpl', step: '14', title: 'TODOS.md' },
|
||||
{
|
||||
file: 'scripts/resolvers/constants.ts', step: '18',
|
||||
historical: 'source comment recording where the third foreground-dispatch recurrence happened (#2440 class); the documentation dispatch now lives at /ship Step 14.5, so this is incident history, not an instruction',
|
||||
},
|
||||
];
|
||||
const SHIP_REFERENCE = /\/ship`?(?:'s)? Step (\d+(?:\.\d+)?)\b/g;
|
||||
|
||||
function shipSteps(): Map<string, string> {
|
||||
const steps = new Map<string, string>();
|
||||
for (const rel of ['ship/SKILL.md.tmpl', ...walk('ship/sections', file => file.endsWith('.md.tmpl'))]) {
|
||||
for (const match of read(rel).matchAll(/^#{2,4} Step (\d+(?:\.\d+)?): (.+)$/gm)) steps.set(match[1]!, match[2]!.trim());
|
||||
}
|
||||
return steps;
|
||||
}
|
||||
|
||||
function shipReferenceProblems(sources: Record<string, string>): string[] {
|
||||
const steps = shipSteps();
|
||||
const problems: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const [file, text] of Object.entries(sources)) {
|
||||
for (const match of text.matchAll(SHIP_REFERENCE)) {
|
||||
const step = match[1]!;
|
||||
const line = lineOf(text, match.index!);
|
||||
const row = SHIP_STEP_REFERENCES.find(r => r.file === file && r.step === step);
|
||||
if (!row) {
|
||||
const near = SHIP_STEP_REFERENCES.filter(r => r.file === file).map(r => `Step ${r.step} (${r.title ?? 'historical'})`);
|
||||
problems.push(`${file}:${line} references /ship Step ${step} ("${steps.get(step) ?? 'undefined'}"), but the table expects ${near.length ? near.join(', ') : 'no /ship reference here'}; fix: correct the step number, or add a row naming the step's title subject`);
|
||||
continue;
|
||||
}
|
||||
seen.add(`${file}#${step}`);
|
||||
if (row.historical) continue;
|
||||
const title = steps.get(step);
|
||||
if (!title) {
|
||||
problems.push(`${file}:${line} references /ship Step ${step}, which ship/SKILL.md.tmpl and ship/sections/*.md.tmpl do not define; fix: point at the step that now owns "${row.title}"`);
|
||||
continue;
|
||||
}
|
||||
if (!title.includes(row.title!)) {
|
||||
problems.push(`${file}:${line} references /ship Step ${step} expecting "${row.title}", but that step is now "${title}"; fix: point at the step titled "${row.title}" or update the row`);
|
||||
continue;
|
||||
}
|
||||
const heading = text.slice(0, match.index!).match(/^#{1,4} .+$/gm)?.at(-1) ?? '';
|
||||
const context = `${heading}\n${text.split('\n')[line - 1]}`;
|
||||
if (!context.includes(row.title!)) {
|
||||
problems.push(`${file}:${line} references /ship Step ${step} ("${title}") but neither the line nor its heading "${heading}" mentions "${row.title}"; fix: reference the /ship step that matches this section's subject`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const row of SHIP_STEP_REFERENCES) {
|
||||
if (row.file in sources && !seen.has(`${row.file}#${row.step}`)) {
|
||||
problems.push(`${row.file}: table row for /ship Step ${row.step} (${row.title ?? 'historical'}) matches no reference; fix: delete the stale row`);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
describe('/ship step references resolve to the step they mean', () => {
|
||||
const sources = Object.fromEntries([...templates, ...resolvers].map(rel => [rel, read(rel)]));
|
||||
|
||||
test('every explicit /ship Step reference names a defined /ship step with the expected title', () => {
|
||||
expect(shipReferenceProblems(sources)).toEqual([]);
|
||||
});
|
||||
|
||||
test('negative control: moving a reference to another existing /ship step fails', () => {
|
||||
const file = 'document-release/sections/release-body.md.tmpl';
|
||||
const moved = sources[file]!.replace("`/ship`'s Step 14.", "`/ship`'s Step 12.");
|
||||
expect(moved).not.toBe(sources[file]);
|
||||
expect(shipSteps().has('12')).toBe(true);
|
||||
const problems = shipReferenceProblems({ ...sources, [file]: moved });
|
||||
expect(problems.some(problem => problem.startsWith(`${file}:`) && problem.includes('Step 12'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// Every literal default-root state path in templates and resolvers. Writers
|
||||
// resolve $GSTACK_STATE_ROOT (docs/state-root.md); a literal ~/.gstack path
|
||||
// only matches them when the state root is the default. Rows are either
|
||||
// user-facing descriptions, manifest/legacy forms read literally, or
|
||||
// conversions still pending (tracked in TODOS.md). New hits fail.
|
||||
const STATE_ROOT_LITERAL = /~\/\.gstack\/(?:projects|analytics)\//g;
|
||||
const RATCHET_ALLOWLIST = JSON.parse(read('test/state-root-ratchet.allowlist.json')) as Array<{ path: string; match: string; reason: string }>;
|
||||
const readinessEvalReason = RATCHET_ALLOWLIST.find(entry => entry.path === 'land-and-deploy/sections/readiness-gate.md.tmpl')!.reason;
|
||||
const MANIFEST_GLOB = 'gbrain context_queries manifest glob: bin/gstack-brain-context-load.ts reads the legacy `~` form literally (default state root only); `{gstack_state_root}/` is the state-root form. Conversion pending (TODOS.md: state-root literals in templates)';
|
||||
const DESCRIPTION = 'user-facing description of where the data lives; no command reads or writes this literal';
|
||||
const PENDING = 'agent reads or writes this literal default-root path; it diverges from $GSTACK_STATE_ROOT writers under a non-default state root. Conversion pending (TODOS.md: state-root literals in templates)';
|
||||
const SOURCE_COMMENT = 'source comment, not rendered into any skill';
|
||||
|
||||
const STATE_ROOT_ALLOWLIST: Array<{ file: string; match: string; reason: string }> = [
|
||||
{ file: 'autoplan/SKILL.md.tmpl', match: 'ls -t ~/.gstack/projects/$SLUG/*-design-*.md', reason: PENDING },
|
||||
{ file: 'autoplan/SKILL.md.tmpl', match: 'test plan on disk at ~/.gstack/projects/$SLUG/', reason: DESCRIPTION },
|
||||
{ file: 'canary/SKILL.md.tmpl', match: 'Append it to `~/.gstack/projects/$SLUG/canary-history.jsonl`', reason: PENDING },
|
||||
{ file: 'careful/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/careful-patterns.txt` (per-project)', reason: DESCRIPTION },
|
||||
{ file: 'design-consultation/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/*-design-*.md"', reason: MANIFEST_GLOB },
|
||||
{ file: 'design-shotgun/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/*-design-*.md"', reason: MANIFEST_GLOB },
|
||||
{ file: 'design-shotgun/SKILL.md.tmpl', match: 'v1 schema at `~/.gstack/projects/$SLUG/taste-profile.json`', reason: DESCRIPTION },
|
||||
{ file: 'health/SKILL.md.tmpl', match: '`~/.gstack/projects/$SLUG/health-history.jsonl`. Zero-check', reason: PENDING },
|
||||
{ file: 'health/SKILL.md.tmpl', match: 'Read the last 10 entries from `~/.gstack/projects/$SLUG/health-history.jsonl`', reason: PENDING },
|
||||
{ file: 'investigate/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/learnings.jsonl"', reason: MANIFEST_GLOB },
|
||||
{ file: 'investigate/SKILL.md.tmpl', match: 'glob: "~/.gstack/analytics/eureka.jsonl"', reason: MANIFEST_GLOB },
|
||||
{ file: 'ios-design-review/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/ios-design-review-<date>.md`', reason: PENDING },
|
||||
{ file: 'land-and-deploy/sections/readiness-gate.md.tmpl', match: 'eval store (`~/.gstack/projects/<slug>/evals/`', reason: readinessEvalReason },
|
||||
{ file: 'land-and-deploy/sections/readiness-gate.md.tmpl', match: 'EVAL_DIR=~/.gstack/projects/$SLUG/evals', reason: readinessEvalReason },
|
||||
{ file: 'office-hours/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/*-design-*.md"', reason: MANIFEST_GLOB },
|
||||
{ file: 'office-hours/SKILL.md.tmpl', match: 'glob: "~/.gstack/analytics/eureka.jsonl"', reason: MANIFEST_GLOB },
|
||||
{ file: 'office-hours/SKILL.md.tmpl', match: "see each other's design docs in `~/.gstack/projects/`", reason: DESCRIPTION },
|
||||
{ file: 'office-hours/sections/design-and-handoff.md.tmpl', match: 'The design doc at `~/.gstack/projects/` is automatically discoverable', reason: DESCRIPTION },
|
||||
{ file: 'plan-ceo-review/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/*-design-*.md"', reason: MANIFEST_GLOB },
|
||||
{ file: 'plan-ceo-review/sections/review-sections.md.tmpl', match: 'Keep in `~/.gstack/projects/` only (local, personal reference)', reason: DESCRIPTION },
|
||||
{ file: 'plan-tune/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/distillation-proposals.json` exists', reason: PENDING },
|
||||
{ file: 'plan-tune/SKILL.md.tmpl', match: 'Logs stay local (`~/.gstack/projects/<slug>/question-log.jsonl`)', reason: DESCRIPTION },
|
||||
{ file: 'plan-tune/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/question-log.jsonl` — nothing leaves your', reason: DESCRIPTION },
|
||||
{ file: 'retro/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/timeline.jsonl"', reason: MANIFEST_GLOB },
|
||||
{ file: 'retro/SKILL.md.tmpl', match: 'glob: "~/.gstack/projects/{repo_slug}/learnings.jsonl"', reason: MANIFEST_GLOB },
|
||||
{ file: 'retro/SKILL.md.tmpl', match: 'Read `~/.gstack/analytics/skill-usage.jsonl`', reason: PENDING },
|
||||
{ file: 'retro/SKILL.md.tmpl', match: 'Read `~/.gstack/analytics/eureka.jsonl`', reason: PENDING },
|
||||
{ file: 'ship/sections/apple-release.md.tmpl', match: 'to `~/.gstack/projects/$SLUG/apple-effects.log`', reason: PENDING },
|
||||
{ file: 'ship/sections/tests.md.tmpl', match: '`~/.gstack/projects/<slug>/<branch>-evidence.jsonl` — Step 16 cites this', reason: PENDING },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '* under ~/.gstack/projects/<slug>/, falling back', reason: SOURCE_COMMENT },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '_LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-$BRANCH-design-*.md', reason: PENDING },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '_LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-design-*.md', reason: PENDING },
|
||||
{ file: 'scripts/resolvers/design.ts', match: 'Glob \\`~/.gstack/projects/$SLUG/designs/**/approved.json\\`', reason: PENDING },
|
||||
{ file: 'scripts/resolvers/learnings.ts', match: '* Learnings are stored per-project at ~/.gstack/projects/{slug}/learnings.jsonl', reason: SOURCE_COMMENT },
|
||||
{ file: 'scripts/resolvers/preamble.ts', match: 'local JSONL append to ~/.gstack/analytics/ (inline, inspectable)', reason: SOURCE_COMMENT },
|
||||
{ file: 'scripts/resolvers/preamble/generate-completion-status.ts', match: '\\`~/.gstack/analytics/\\`, matching preamble analytics writes', reason: PENDING },
|
||||
];
|
||||
|
||||
describe('templates and resolvers name gstack state only through allowlisted literals', () => {
|
||||
const hits = [...templates, ...resolvers].flatMap(file => {
|
||||
const text = read(file);
|
||||
return [...text.matchAll(STATE_ROOT_LITERAL)].map(match => {
|
||||
const line = lineOf(text, match.index!);
|
||||
return { file, line, text: text.split('\n')[line - 1]! };
|
||||
});
|
||||
});
|
||||
|
||||
test('every ~/.gstack/projects/ and ~/.gstack/analytics/ hit is allowlisted with a reason', () => {
|
||||
expect(hits.length).toBeGreaterThan(0);
|
||||
const unlisted = hits
|
||||
.filter(hit => !STATE_ROOT_ALLOWLIST.some(row => row.file === hit.file && row.reason && hit.text.includes(row.match)))
|
||||
.map(hit => `${hit.file}:${hit.line} names ${hit.text.match(STATE_ROOT_LITERAL)![0]} ("${hit.text.trim().slice(0, 100)}"); fix: use $GSTACK_STATE_ROOT (docs/state-root.md) or add a STATE_ROOT_ALLOWLIST row with a reason`);
|
||||
expect(unlisted).toEqual([]);
|
||||
});
|
||||
|
||||
test('every state-root allowlist row still matches a hit', () => {
|
||||
const stale = STATE_ROOT_ALLOWLIST
|
||||
.filter(row => !hits.some(hit => hit.file === row.file && hit.text.includes(row.match)))
|
||||
.map(row => `${row.file}: allowlist row "${row.match}" matches no ~/.gstack/projects|analytics line; fix: delete the row (the literal was converted or moved)`);
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user