W7 T8: bun run audit:manifest and the prompt-audit procedure

scripts/prompt-audit-manifest.ts prints the slices a contributor feeds to
/claude-api prompt-audit: shared instructions (CLAUDE.md, overlays, preamble
resolvers), resolvers, skill templates (a skill stays in one slice),
model-request code and tests that pin skill wording. CONTRIBUTING.md gains the
procedure; CLAUDE.md lists the command with its frontier-release cadence.

CLAUDE.md

@@ -31,6 +31,7 @@ bun run eval:summary # aggregate stats across all eval runs
 bun run eval:flake-rank  # rank tests by flake signal (retried passes first; --json, --dir, --since-days)
 bun run slop          # full slop-scan report (all files)
 bun run slop:diff     # slop findings in files changed on this branch only
+bun run audit:manifest  # file slices for /claude-api prompt-audit; rerun it at each frontier-model release (CONTRIBUTING.md)
 ```
 
 `test:evals` requires `ANTHROPIC_API_KEY`. Codex E2E tests (`test/codex-e2e.test.ts`,

CONTRIBUTING.md

@@ -603,6 +603,31 @@ is a CLI flag) so every caller gets it on both paths. Exported test seams:
 `serveDir(root, nonce)`, `SAFE_TMP_DIR`, and `PAGE_NUMBER_FOOTER` (the one
 page-number footer make-pdf, `gstack-render`, and the browse `pdf` command share).
 
+## Prompt audit at each frontier-model release
+
+When a new frontier Claude model ships, audit the text models read for
+instructions the new model over-applies or no longer needs. The audit is
+Anthropic's `/claude-api prompt-audit`, a Claude Code skill you run in your own
+Claude Code session; gstack only prints what to feed it.
+
+```bash
+bun run audit:manifest   # slices of templates, resolvers, overlays, CLAUDE.md and wording-pinning tests (--json for a machine-readable list)
+```
+
+1. In Claude Code, run `/claude-api prompt-audit` with the new model as the
+   target, one slice at a time, giving it that slice's file list. `s01` is the
+   shared text every skill reads (CLAUDE.md, model overlays, preamble
+   resolvers); a skill's template and its sections share one slice.
+2. Fix findings in templates and resolvers, never in generated SKILL.md files,
+   then run `bun run gen:skill-docs --host all`.
+3. Treat safety rules as held: reword one only where an eval shows the model
+   obeys it both before and after the change.
+4. Work through the "tests that pin skill wording" slices last. Where prose
+   changed, replace exact-sentence pins with structural or meaning checks
+   (the prompt-bytes rule in CLAUDE.md's testing section).
+5. `test/archaeology-lint.test.ts` keeps issue numbers and incident stories out
+   of the generated text; its failure names the source file to fix.
+
 ## Jargon list (V1 writing style)
 
 gstack's Writing Style section (injected into every tier-≥2 skill's preamble)

package.json

@@ -25,6 +25,7 @@
     "build:gates": "bun build --compile make-pdf/src/cli.ts --outfile make-pdf/dist/pdf && bun build --compile browse/src/cli.ts --outfile browse/dist/browse && bun run build:diagram-render",
     "gen:skill-docs": "bun run scripts/gen-skill-docs.ts",
     "gen:skill-docs:user": "bun run scripts/gen-skill-docs.ts --respect-detection",
+    "audit:manifest": "bun run scripts/prompt-audit-manifest.ts",
     "dev": "bun run browse/src/cli.ts",
     "server": "bun run browse/src/server.ts",
     "test": "bun run scripts/test-free-shards.ts",

scripts/prompt-audit-manifest.ts

@@ -0,0 +1,114 @@
+#!/usr/bin/env bun
+/**
+ * Prompt-audit manifest: the files a contributor feeds to Anthropic's
+ * `/claude-api prompt-audit` (a Claude Code skill, run in your own Claude Code
+ * session) at each frontier-model release, cut into slices small enough for
+ * one audit pass each. Procedure: CONTRIBUTING.md "Prompt audit at each
+ * frontier-model release".
+ *
+ * Slices, in order: shared instructions every skill reads (CLAUDE.md, model
+ * overlays, preamble resolvers), the other resolvers, the skill templates
+ * (a skill's SKILL.md.tmpl and sections stay in one slice), the code that
+ * builds model requests, and the tests that pin skill wording (they read a
+ * template, skill or resolver and assert a literal of 40+ characters).
+ *
+ * Usage: bun run audit:manifest [--json] [--budget-kb <n>]
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
+
+const ROOT = resolve(import.meta.dir, '..');
+const DEFAULT_BUDGET_KB = 200;
+
+export interface AuditSlice { id: string; group: string; files: string[]; bytes: number }
+
+function walk(dir: string, keep: (rel: string) => boolean): string[] {
+  const out: string[] = [];
+  const visit = (d: string) => {
+    for (const entry of readdirSync(d, { withFileTypes: true })) {
+      const p = join(d, entry.name);
+      const rel = relative(ROOT, p);
+      if (entry.isDirectory()) {
+        if (!entry.name.startsWith('.') && entry.name !== 'node_modules') visit(p);
+      } else if (keep(rel)) {
+        out.push(rel);
+      }
+    }
+  };
+  visit(dir);
+  return out.sort();
+}
+
+const size = (rel: string) => statSync(join(ROOT, rel)).size;
+const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8');
+
+/** Pack units (each a list of files that must stay together) into slices under the budget. */
+function pack(group: string, units: string[][], budget: number): Omit<AuditSlice, 'id'>[] {
+  const slices: Omit<AuditSlice, 'id'>[] = [];
+  let current: Omit<AuditSlice, 'id'> | null = null;
+  for (const unit of units) {
+    const bytes = unit.reduce((sum, f) => sum + size(f), 0);
+    if (!current || (current.bytes > 0 && current.bytes + bytes > budget)) {
+      current = { group, files: [], bytes: 0 };
+      slices.push(current);
+    }
+    current.files.push(...unit);
+    current.bytes += bytes;
+  }
+  return slices;
+}
+
+export function buildManifest(budgetKb = DEFAULT_BUDGET_KB): AuditSlice[] {
+  const budget = budgetKb * 1024;
+  const preamble = walk(join(ROOT, 'scripts/resolvers/preamble'), rel => rel.endsWith('.ts'));
+  const shared = ['CLAUDE.md', ...walk(join(ROOT, 'model-overlays'), rel => rel.endsWith('.md')), ...preamble];
+  const resolvers = walk(join(ROOT, 'scripts/resolvers'), rel => rel.endsWith('.ts') && !preamble.includes(rel));
+
+  const templates = walk(ROOT, rel => rel.endsWith('.tmpl') && !rel.startsWith('test/'));
+  const bySkill = new Map<string, string[]>();
+  for (const rel of templates) {
+    const skill = rel.includes('/') ? rel.split('/')[0] : '(root)';
+    bySkill.set(skill, [...(bySkill.get(skill) ?? []), rel]);
+  }
+
+  const requestCode = [...walk(join(ROOT, 'lib'), rel => rel.endsWith('.ts')),
+    ...walk(join(ROOT, 'design/src'), rel => rel.endsWith('.ts')),
+    ...walk(join(ROOT, 'test/helpers'), rel => rel.endsWith('.ts'))]
+    .filter(rel => /api\.anthropic\.com|api\.openai\.com|\/v1\/messages|messages\.create|responses\.create/.test(read(rel)));
+
+  const pinningTests = walk(join(ROOT, 'test'), rel => /^test\/[^/]+\.test\.ts$/.test(rel))
+    .filter(rel => {
+      const src = read(rel);
+      return /SKILL\.md|\.md\.tmpl|scripts\/resolvers|model-overlays/.test(src) && /\.(toContain|toMatch)\((['"`])[^'"`]{40,}/.test(src);
+    });
+
+  const slices = [
+    ...pack('shared instructions', [shared], Infinity),
+    ...pack('resolvers', resolvers.map(f => [f]), budget),
+    ...pack('skill templates', [...bySkill.values()], budget),
+    ...pack('model-request code', requestCode.map(f => [f]), budget),
+    ...pack('tests that pin skill wording', pinningTests.map(f => [f]), budget),
+  ];
+  return slices.map((slice, i) => ({ id: `s${String(i + 1).padStart(2, '0')}`, ...slice }));
+}
+
+if (import.meta.main) {
+  const args = process.argv.slice(2);
+  const budgetArg = args.indexOf('--budget-kb');
+  const budgetKb = budgetArg >= 0 ? Number(args[budgetArg + 1]) : DEFAULT_BUDGET_KB;
+  if (!Number.isFinite(budgetKb) || budgetKb <= 0) {
+    console.error('--budget-kb needs a positive number of kilobytes');
+    process.exit(2);
+  }
+  const manifest = buildManifest(budgetKb);
+  if (args.includes('--json')) {
+    console.log(JSON.stringify(manifest, null, 2));
+  } else {
+    const files = manifest.reduce((n, s) => n + s.files.length, 0);
+    console.log(`Prompt-audit manifest: ${manifest.length} slices, ${files} files. Run /claude-api prompt-audit in Claude Code on one slice at a time (CONTRIBUTING.md).`);
+    for (const slice of manifest) {
+      console.log(`\n## ${slice.id} ${slice.group} (${slice.files.length} files, ${Math.round(slice.bytes / 1024)} KB)`);
+      for (const file of slice.files) console.log(file);
+    }
+  }
+}

test/prompt-audit-manifest.test.ts

@@ -0,0 +1,51 @@
+/**
+ * `bun run audit:manifest` — the slices a contributor feeds to
+ * `/claude-api prompt-audit` at each frontier-model release.
+ */
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import { readdirSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { buildManifest } from '../scripts/prompt-audit-manifest';
+
+const ROOT = resolve(import.meta.dir, '..');
+
+describe('prompt-audit manifest', () => {
+  const manifest = buildManifest();
+  const files = manifest.flatMap(s => s.files);
+
+  test('covers CLAUDE.md, every overlay and resolver, and the skill templates, each once', () => {
+    expect(new Set(files).size).toBe(files.length);
+    expect(files).toContain('CLAUDE.md');
+    for (const overlay of readdirSync(join(ROOT, 'model-overlays'))) expect(files).toContain(`model-overlays/${overlay}`);
+    for (const rel of ['scripts/resolvers/constants.ts', 'scripts/resolvers/preamble.ts', 'ship/SKILL.md.tmpl', 'codex/sections/consult-mode.md.tmpl', 'test/helpers/llm-judge.ts']) {
+      expect(files).toContain(rel);
+    }
+    expect(files.some(f => f.endsWith('SKILL.md'))).toBe(false);
+    const resolvers = (readdirSync(join(ROOT, 'scripts/resolvers'), { recursive: true }) as string[])
+      .filter(f => f.endsWith('.ts')).map(f => `scripts/resolvers/${f}`);
+    expect(resolvers.filter(f => !files.includes(f))).toEqual([]);
+  });
+
+  test('a skill stays in one slice, and only a single oversized unit exceeds the budget', () => {
+    const sliceOf = new Map(manifest.flatMap(s => s.files.map(f => [f, s.id] as const)));
+    expect(sliceOf.get('codex/SKILL.md.tmpl')).toBe(sliceOf.get('codex/sections/consult-mode.md.tmpl'));
+    for (const slice of manifest.filter(s => s.group !== 'shared instructions')) {
+      if (slice.bytes > 200 * 1024) {
+        const skills = new Set(slice.files.map(f => f.split('/')[0]));
+        expect(slice.files.length === 1 || skills.size === 1).toBe(true);
+      }
+      expect(slice.bytes).toBe(slice.files.reduce((n, f) => n + statSync(join(ROOT, f)).size, 0));
+    }
+    expect(manifest.map(s => s.id)).toEqual(manifest.map((_, i) => `s${String(i + 1).padStart(2, '0')}`));
+  });
+
+  test('bun run audit:manifest prints the slices', () => {
+    const r = spawnSync('bun', ['run', 'audit:manifest'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
+    expect(r.status).toBe(0);
+    expect(r.stdout).toContain('/claude-api prompt-audit');
+    expect(r.stdout).toContain('## s01 shared instructions');
+    const json = spawnSync('bun', ['run', 'scripts/prompt-audit-manifest.ts', '--json'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
+    expect(JSON.parse(json.stdout)).toEqual(manifest);
+  });
+});