From 3fa20ee5f789fea8bad0b2ea14e0c99593186de9 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 20:55:04 +0000 Subject: [PATCH] W7 T8: bun run audit:manifest and the prompt-audit procedure scripts/prompt-audit-manifest.ts prints the slices a contributor feeds to /claude-api prompt-audit: shared instructions (CLAUDE.md, overlays, preamble resolvers), resolvers, skill templates (a skill stays in one slice), model-request code and tests that pin skill wording. CONTRIBUTING.md gains the procedure; CLAUDE.md lists the command with its frontier-release cadence. --- CLAUDE.md | 1 + CONTRIBUTING.md | 25 +++++++ package.json | 1 + scripts/prompt-audit-manifest.ts | 114 +++++++++++++++++++++++++++++ test/prompt-audit-manifest.test.ts | 51 +++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 scripts/prompt-audit-manifest.ts create mode 100644 test/prompt-audit-manifest.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 45c475642..1875cb940 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,6 +31,7 @@ bun run eval:summary # aggregate stats across all eval runs bun run eval:flake-rank # rank tests by flake signal (retried passes first; --json, --dir, --since-days) bun run slop # full slop-scan report (all files) bun run slop:diff # slop findings in files changed on this branch only +bun run audit:manifest # file slices for /claude-api prompt-audit; rerun it at each frontier-model release (CONTRIBUTING.md) ``` `test:evals` requires `ANTHROPIC_API_KEY`. Codex E2E tests (`test/codex-e2e.test.ts`, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83b605ffb..09ff72c69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -603,6 +603,31 @@ is a CLI flag) so every caller gets it on both paths. Exported test seams: `serveDir(root, nonce)`, `SAFE_TMP_DIR`, and `PAGE_NUMBER_FOOTER` (the one page-number footer make-pdf, `gstack-render`, and the browse `pdf` command share). +## Prompt audit at each frontier-model release + +When a new frontier Claude model ships, audit the text models read for +instructions the new model over-applies or no longer needs. The audit is +Anthropic's `/claude-api prompt-audit`, a Claude Code skill you run in your own +Claude Code session; gstack only prints what to feed it. + +```bash +bun run audit:manifest # slices of templates, resolvers, overlays, CLAUDE.md and wording-pinning tests (--json for a machine-readable list) +``` + +1. In Claude Code, run `/claude-api prompt-audit` with the new model as the + target, one slice at a time, giving it that slice's file list. `s01` is the + shared text every skill reads (CLAUDE.md, model overlays, preamble + resolvers); a skill's template and its sections share one slice. +2. Fix findings in templates and resolvers, never in generated SKILL.md files, + then run `bun run gen:skill-docs --host all`. +3. Treat safety rules as held: reword one only where an eval shows the model + obeys it both before and after the change. +4. Work through the "tests that pin skill wording" slices last. Where prose + changed, replace exact-sentence pins with structural or meaning checks + (the prompt-bytes rule in CLAUDE.md's testing section). +5. `test/archaeology-lint.test.ts` keeps issue numbers and incident stories out + of the generated text; its failure names the source file to fix. + ## Jargon list (V1 writing style) gstack's Writing Style section (injected into every tier-≥2 skill's preamble) diff --git a/package.json b/package.json index 1132252e6..9a28bc6fd 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "build:gates": "bun build --compile make-pdf/src/cli.ts --outfile make-pdf/dist/pdf && bun build --compile browse/src/cli.ts --outfile browse/dist/browse && bun run build:diagram-render", "gen:skill-docs": "bun run scripts/gen-skill-docs.ts", "gen:skill-docs:user": "bun run scripts/gen-skill-docs.ts --respect-detection", + "audit:manifest": "bun run scripts/prompt-audit-manifest.ts", "dev": "bun run browse/src/cli.ts", "server": "bun run browse/src/server.ts", "test": "bun run scripts/test-free-shards.ts", diff --git a/scripts/prompt-audit-manifest.ts b/scripts/prompt-audit-manifest.ts new file mode 100644 index 000000000..e52035220 --- /dev/null +++ b/scripts/prompt-audit-manifest.ts @@ -0,0 +1,114 @@ +#!/usr/bin/env bun +/** + * Prompt-audit manifest: the files a contributor feeds to Anthropic's + * `/claude-api prompt-audit` (a Claude Code skill, run in your own Claude Code + * session) at each frontier-model release, cut into slices small enough for + * one audit pass each. Procedure: CONTRIBUTING.md "Prompt audit at each + * frontier-model release". + * + * Slices, in order: shared instructions every skill reads (CLAUDE.md, model + * overlays, preamble resolvers), the other resolvers, the skill templates + * (a skill's SKILL.md.tmpl and sections stay in one slice), the code that + * builds model requests, and the tests that pin skill wording (they read a + * template, skill or resolver and assert a literal of 40+ characters). + * + * Usage: bun run audit:manifest [--json] [--budget-kb ] + */ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative, resolve } from 'node:path'; + +const ROOT = resolve(import.meta.dir, '..'); +const DEFAULT_BUDGET_KB = 200; + +export interface AuditSlice { id: string; group: string; files: string[]; bytes: number } + +function walk(dir: string, keep: (rel: string) => boolean): string[] { + const out: string[] = []; + const visit = (d: string) => { + for (const entry of readdirSync(d, { withFileTypes: true })) { + const p = join(d, entry.name); + const rel = relative(ROOT, p); + if (entry.isDirectory()) { + if (!entry.name.startsWith('.') && entry.name !== 'node_modules') visit(p); + } else if (keep(rel)) { + out.push(rel); + } + } + }; + visit(dir); + return out.sort(); +} + +const size = (rel: string) => statSync(join(ROOT, rel)).size; +const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8'); + +/** Pack units (each a list of files that must stay together) into slices under the budget. */ +function pack(group: string, units: string[][], budget: number): Omit[] { + const slices: Omit[] = []; + let current: Omit | null = null; + for (const unit of units) { + const bytes = unit.reduce((sum, f) => sum + size(f), 0); + if (!current || (current.bytes > 0 && current.bytes + bytes > budget)) { + current = { group, files: [], bytes: 0 }; + slices.push(current); + } + current.files.push(...unit); + current.bytes += bytes; + } + return slices; +} + +export function buildManifest(budgetKb = DEFAULT_BUDGET_KB): AuditSlice[] { + const budget = budgetKb * 1024; + const preamble = walk(join(ROOT, 'scripts/resolvers/preamble'), rel => rel.endsWith('.ts')); + const shared = ['CLAUDE.md', ...walk(join(ROOT, 'model-overlays'), rel => rel.endsWith('.md')), ...preamble]; + const resolvers = walk(join(ROOT, 'scripts/resolvers'), rel => rel.endsWith('.ts') && !preamble.includes(rel)); + + const templates = walk(ROOT, rel => rel.endsWith('.tmpl') && !rel.startsWith('test/')); + const bySkill = new Map(); + for (const rel of templates) { + const skill = rel.includes('/') ? rel.split('/')[0] : '(root)'; + bySkill.set(skill, [...(bySkill.get(skill) ?? []), rel]); + } + + const requestCode = [...walk(join(ROOT, 'lib'), rel => rel.endsWith('.ts')), + ...walk(join(ROOT, 'design/src'), rel => rel.endsWith('.ts')), + ...walk(join(ROOT, 'test/helpers'), rel => rel.endsWith('.ts'))] + .filter(rel => /api\.anthropic\.com|api\.openai\.com|\/v1\/messages|messages\.create|responses\.create/.test(read(rel))); + + const pinningTests = walk(join(ROOT, 'test'), rel => /^test\/[^/]+\.test\.ts$/.test(rel)) + .filter(rel => { + const src = read(rel); + return /SKILL\.md|\.md\.tmpl|scripts\/resolvers|model-overlays/.test(src) && /\.(toContain|toMatch)\((['"`])[^'"`]{40,}/.test(src); + }); + + const slices = [ + ...pack('shared instructions', [shared], Infinity), + ...pack('resolvers', resolvers.map(f => [f]), budget), + ...pack('skill templates', [...bySkill.values()], budget), + ...pack('model-request code', requestCode.map(f => [f]), budget), + ...pack('tests that pin skill wording', pinningTests.map(f => [f]), budget), + ]; + return slices.map((slice, i) => ({ id: `s${String(i + 1).padStart(2, '0')}`, ...slice })); +} + +if (import.meta.main) { + const args = process.argv.slice(2); + const budgetArg = args.indexOf('--budget-kb'); + const budgetKb = budgetArg >= 0 ? Number(args[budgetArg + 1]) : DEFAULT_BUDGET_KB; + if (!Number.isFinite(budgetKb) || budgetKb <= 0) { + console.error('--budget-kb needs a positive number of kilobytes'); + process.exit(2); + } + const manifest = buildManifest(budgetKb); + if (args.includes('--json')) { + console.log(JSON.stringify(manifest, null, 2)); + } else { + const files = manifest.reduce((n, s) => n + s.files.length, 0); + console.log(`Prompt-audit manifest: ${manifest.length} slices, ${files} files. Run /claude-api prompt-audit in Claude Code on one slice at a time (CONTRIBUTING.md).`); + for (const slice of manifest) { + console.log(`\n## ${slice.id} ${slice.group} (${slice.files.length} files, ${Math.round(slice.bytes / 1024)} KB)`); + for (const file of slice.files) console.log(file); + } + } +} diff --git a/test/prompt-audit-manifest.test.ts b/test/prompt-audit-manifest.test.ts new file mode 100644 index 000000000..e376b9efe --- /dev/null +++ b/test/prompt-audit-manifest.test.ts @@ -0,0 +1,51 @@ +/** + * `bun run audit:manifest` — the slices a contributor feeds to + * `/claude-api prompt-audit` at each frontier-model release. + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { readdirSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { buildManifest } from '../scripts/prompt-audit-manifest'; + +const ROOT = resolve(import.meta.dir, '..'); + +describe('prompt-audit manifest', () => { + const manifest = buildManifest(); + const files = manifest.flatMap(s => s.files); + + test('covers CLAUDE.md, every overlay and resolver, and the skill templates, each once', () => { + expect(new Set(files).size).toBe(files.length); + expect(files).toContain('CLAUDE.md'); + for (const overlay of readdirSync(join(ROOT, 'model-overlays'))) expect(files).toContain(`model-overlays/${overlay}`); + for (const rel of ['scripts/resolvers/constants.ts', 'scripts/resolvers/preamble.ts', 'ship/SKILL.md.tmpl', 'codex/sections/consult-mode.md.tmpl', 'test/helpers/llm-judge.ts']) { + expect(files).toContain(rel); + } + expect(files.some(f => f.endsWith('SKILL.md'))).toBe(false); + const resolvers = (readdirSync(join(ROOT, 'scripts/resolvers'), { recursive: true }) as string[]) + .filter(f => f.endsWith('.ts')).map(f => `scripts/resolvers/${f}`); + expect(resolvers.filter(f => !files.includes(f))).toEqual([]); + }); + + test('a skill stays in one slice, and only a single oversized unit exceeds the budget', () => { + const sliceOf = new Map(manifest.flatMap(s => s.files.map(f => [f, s.id] as const))); + expect(sliceOf.get('codex/SKILL.md.tmpl')).toBe(sliceOf.get('codex/sections/consult-mode.md.tmpl')); + for (const slice of manifest.filter(s => s.group !== 'shared instructions')) { + if (slice.bytes > 200 * 1024) { + const skills = new Set(slice.files.map(f => f.split('/')[0])); + expect(slice.files.length === 1 || skills.size === 1).toBe(true); + } + expect(slice.bytes).toBe(slice.files.reduce((n, f) => n + statSync(join(ROOT, f)).size, 0)); + } + expect(manifest.map(s => s.id)).toEqual(manifest.map((_, i) => `s${String(i + 1).padStart(2, '0')}`)); + }); + + test('bun run audit:manifest prints the slices', () => { + const r = spawnSync('bun', ['run', 'audit:manifest'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(r.status).toBe(0); + expect(r.stdout).toContain('/claude-api prompt-audit'); + expect(r.stdout).toContain('## s01 shared instructions'); + const json = spawnSync('bun', ['run', 'scripts/prompt-audit-manifest.ts', '--json'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(JSON.parse(json.stdout)).toEqual(manifest); + }); +});