/** * The preamble's Claims Need Evidence kernel (docs/designs/HONEST_WORK_GATE_INTEGRITY.md W4). * * Pins where the kernel renders (tiers 2–4, default and terse; never tier 1), * its byte budget, and the machine-read markers each bullet carries; never the * sentences (docs/test-value-bar.md). The behavior claim is carried by the * selected prompt evals, not here. */ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { readSkillStartup } from './helpers/skill-startup'; import { generatePreamble } from '../scripts/resolvers/preamble'; import { EVIDENCE_KERNEL_BULLETS, EVIDENCE_KERNEL_HEADING, KERNEL_MAX_BYTES, generateEvidenceDirective, } from '../scripts/resolvers/preamble/generate-evidence-directive'; import { generateCompletionStatus } from '../scripts/resolvers/preamble/generate-completion-status'; import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; import { between, expectMentions, expectTokens } from './helpers/prompt-structure'; const ROOT = path.resolve(import.meta.dir, '..'); const ctx = (tier: number, explainLevel?: 'default' | 'terse'): TemplateContext => ({ skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude, preambleTier: tier, explainLevel, } as TemplateContext); describe('Claims Need Evidence kernel', () => { test('renders five bullets within the byte budget, with the stderr rule and no cross-skill step pointer', () => { const rendered = generateEvidenceDirective(); expect(rendered.startsWith(`${EVIDENCE_KERNEL_HEADING}\n`)).toBe(true); const bullets = rendered.split('\n').filter(line => line.startsWith('- ')); expect(bullets).toHaveLength(5); expect(EVIDENCE_KERNEL_BULLETS).toHaveLength(5); expect(Buffer.byteLength(bullets.join('\n'), 'utf8')).toBeLessThanOrEqual(KERNEL_MAX_BYTES); expectTokens(rendered, ['stderr', '"done, unverified"'], 'kernel'); expect(rendered).not.toMatch(/\bStep \d/); expectMentions(rendered, [ ['limitation', 'verbatim error'], ['execution', 'command', 'fingerprint'], ['mock', 'live check'], ['disclose', 'failure', 'conclusion'], ['null result', 'success'], ], 'kernel'); expect(rendered).not.toContain('Claimed Limitations Need Evidence'); }); test('composes for tiers 2–4 in default and terse builds, never for tier 1', () => { for (const tier of [2, 3, 4]) { for (const level of ['default', 'terse'] as const) { const out = generatePreamble(ctx(tier, level)); expect(out, `tier ${tier} ${level} lacks the kernel`).toContain(generateEvidenceDirective()); expect(out.indexOf(EVIDENCE_KERNEL_HEADING)).toBeLessThan(out.indexOf('## Completion Status Protocol')); } } for (const level of ['default', 'terse'] as const) { expect(generatePreamble(ctx(1, level))).not.toContain(EVIDENCE_KERNEL_HEADING); } }); test('the DONE line binds evidence to the final consumed inputs and names reuse (every tier)', () => { const status = between(generateCompletionStatus(ctx(1)), '## Completion Status Protocol', '## Operational Self-Improvement'); const done = status.split('\n').find(line => line.startsWith('- **DONE**'))!; expectTokens(done, ['**DONE**', 'final consumed inputs', 'reuse'], 'DONE line'); expect(done).not.toBe('- **DONE** — completed with evidence.'); for (const tier of [1, 2, 3, 4]) expect(generatePreamble(ctx(tier))).toContain(done); }); test('generated SKILL.md files carry the kernel at tier 2+ and the old heading nowhere', () => { const skills = fs.readdirSync(ROOT).filter(dir => fs.existsSync(path.join(ROOT, dir, 'SKILL.md.tmpl')) && fs.existsSync(path.join(ROOT, dir, 'SKILL.md'))); expect(skills.length).toBeGreaterThan(20); for (const skill of skills) { const tier = Number(/preamble-tier:\s*(\d)/.exec(fs.readFileSync(path.join(ROOT, skill, 'SKILL.md.tmpl'), 'utf8'))?.[1]); const rendered = readSkillStartup(path.join(ROOT, skill, 'SKILL.md')); expect(rendered, `${skill} still carries the retired heading`).not.toContain('Claimed Limitations Need Evidence'); if (tier >= 2) expect(rendered, `${skill} (tier ${tier}) lacks the kernel; run bun run gen:skill-docs`).toContain(EVIDENCE_KERNEL_HEADING); } }); });