docs+test: AGENTS.md/docs/skills.md inventory sync + private-path leak detector

Inventory sync (codex-flagged drift):
- /debug → /investigate (skill renamed in v1.0.1.0)
- AGENTS.md grows from 21 to 40+ skills, organized by category (plan reviews,
  implementation, release, operational, browser, safety)
- docs/skills.md gains 11 missing entries: /plan-devex-review, /devex-review,
  /plan-tune, /context-save, /context-restore, /health, /landing-report,
  /benchmark-models, /pair-agent, /setup-gbrain, /make-pdf
- Stale "<5s bun test" claim dropped — slim-preamble harness + new tests means
  no realistic universal claim to make
- Adds explicit "Mac + Linux full, curated Windows lane" platform statement +
  "Git Bash / MSYS today, native PowerShell future" install note

New invariants in test/skill-validation.test.ts (~80 LOC):
- Private-path leak detector scans every SKILL.md / SKILL.md.tmpl for known
  maintainer-only filenames (coordination-board.md, SEEKING_LOG.md,
  RATIONAL_SUBJECT.md, VALUE_SIGNAL_LOOP.md, C:\LLM Playground\go).
  Adapted from the McGluut fork's skill-contract-audit.ts; we don't take
  the script wholesale because most of its checks are already covered by
  test/gen-skill-docs.test.ts:1668-2074 and test/skill-validation.test.ts:1419
  — only the private-path scan and doc-inventory cross-check are new.
- Doc-inventory cross-check: every skill directory with a SKILL.md.tmpl must
  appear in both AGENTS.md and docs/skills.md. Catches the inventory drift
  this commit is fixing — without this test it would just drift again.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-04-27 23:01:49 -07:00
parent df9f7b69c9
commit 87ce4c696f
3 changed files with 182 additions and 8 deletions
+101
View File
@@ -1458,6 +1458,107 @@ describe('Skill trigger phrases', () => {
}
});
// ─── Private-path leak detector ──────────────────────────────
//
// Catches accidental references to maintainer-private files in skill output.
// Adapted from the McGluut fork's skill-contract-audit.ts (we don't take the
// whole script — these are the unique checks not already covered by
// test/gen-skill-docs.test.ts:1668-2074 .claude/skills leakage tests).
describe('Private-path leak detection', () => {
const PRIVATE_PATTERNS: Array<{ pattern: RegExp; label: string }> = [
{ pattern: /coordination-board\.md/i, label: 'coordination-board.md' },
{ pattern: /SEEKING_LOG\.md/, label: 'SEEKING_LOG.md' },
{ pattern: /RATIONAL_SUBJECT\.md/, label: 'RATIONAL_SUBJECT.md' },
{ pattern: /VALUE_SIGNAL_LOOP\.md/, label: 'VALUE_SIGNAL_LOOP.md' },
{ pattern: /C:\\\\LLM Playground\\\\go/i, label: 'C:\\LLM Playground\\go' },
];
// Walk every SKILL.md and SKILL.md.tmpl in the repo (excluding node_modules,
// generated host outputs, and .git).
function discoverSkillSurface(): string[] {
const results: string[] = [];
function walk(dir: string) {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
if (entry.name.startsWith('.') && entry.name !== '.agents') continue;
if (entry.name === 'node_modules' || entry.name === 'dist') continue;
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(full);
} else if (entry.name === 'SKILL.md' || entry.name === 'SKILL.md.tmpl') {
results.push(full);
}
}
}
walk(ROOT);
return results;
}
test('no SKILL.md or SKILL.md.tmpl references private maintainer files', () => {
const files = discoverSkillSurface();
expect(files.length).toBeGreaterThan(0);
const leaks: string[] = [];
for (const file of files) {
const content = fs.readFileSync(file, 'utf-8');
for (const { pattern, label } of PRIVATE_PATTERNS) {
if (pattern.test(content)) {
leaks.push(`${path.relative(ROOT, file)} mentions ${label}`);
}
}
}
expect(leaks).toEqual([]);
});
});
// ─── Doc-inventory cross-check ───────────────────────────────
//
// Every skill directory (with a SKILL.md.tmpl) must appear in both AGENTS.md
// and docs/skills.md. Catches the inventory drift codex flagged (/debug
// → /investigate; missing /autoplan, /context-save, /plan-devex-review, etc.).
describe('Doc inventory cross-check', () => {
// Skills that don't get user-invocation lines in agent-facing docs.
// - 'qa-only' is a sub-mode of /qa with shared docs.
// - The 5 listed below are infrastructure (model overlays, shipped binary,
// hosts) that don't show up in the user-facing skill table.
const DOC_INVENTORY_EXCLUDE = new Set([
// Infra / non-skills
'agents', 'claude', 'connect-chrome', 'contrib', 'hosts',
'lib', 'model-overlays', 'openclaw', 'supabase', 'scripts', 'test',
]);
function discoverSkillDirs(): string[] {
const dirs: string[] = [];
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
if (entry.name.startsWith('.')) continue;
if (DOC_INVENTORY_EXCLUDE.has(entry.name)) continue;
const tmplPath = path.join(ROOT, entry.name, 'SKILL.md.tmpl');
if (fs.existsSync(tmplPath)) dirs.push(entry.name);
}
return dirs.sort();
}
test('every skill is documented in AGENTS.md', () => {
const agents = fs.readFileSync(path.join(ROOT, 'AGENTS.md'), 'utf-8');
const missing: string[] = [];
for (const skill of discoverSkillDirs()) {
// Match `/skill-name` as a token boundary.
if (!new RegExp(`/${skill}\\b`).test(agents)) missing.push(skill);
}
expect(missing).toEqual([]);
});
test('every skill is documented in docs/skills.md', () => {
const docs = fs.readFileSync(path.join(ROOT, 'docs', 'skills.md'), 'utf-8');
const missing: string[] = [];
for (const skill of discoverSkillDirs()) {
if (!new RegExp(`/${skill}\\b`).test(docs)) missing.push(skill);
}
expect(missing).toEqual([]);
});
});
// ─── Codex Skill Validation ──────────────────────────────────
describe('Codex skill validation', () => {