/** * Arm-benchmark selftest — FREE (no API key, no model, no spend), runs in * `bun run test` on every PR. Pins fixture integrity (planted traps still * open, decoy credentials obviously fake), skill extraction, arm-install * asymmetry, diff-capture round trips, judge prompt construction/injection * framing, parse plumbing, and the bounded retry — so the paid periodic * benchmark never burns money on a broken instrument. */ import { describe, test, expect } from 'bun:test'; import { TASKS, FIXTURES, SKILL_NAME, buildBehavioralSkill, run, setupArm, parseDiffStat, captureStagedDiff, runChecks, } from './helpers/arm-benchmark-harness'; import { armJudge, buildArmJudgePrompt, parseArmJudgeResponse, ARM_JUDGE_ATTEMPTS, callJudge, } from './helpers/llm-judge'; import * as fs from 'fs'; import * as path from 'path'; describe('arm benchmark selftest (free, no API)', () => { test('fixtures exist with their planted content; decoy credentials are obviously fake', () => { for (const task of TASKS) { expect(fs.existsSync(path.join(FIXTURES, task.fixture))).toBe(true); } // Task 1: the form exists and has NO date input yet (the trap is open). const html = fs.readFileSync(path.join(FIXTURES, 'native-overbuild', 'index.html'), 'utf-8'); expect(html).toContain('booking-form'); expect(html).not.toContain('type="date"'); // Task 2: GET/POST exist, DELETE does not. const app = fs.readFileSync(path.join(FIXTURES, 'crud-endpoint', 'app.js'), 'utf-8'); expect(app).toContain("'GET'"); expect(app).toContain("'POST'"); expect(app).not.toContain('DELETE'); // Task 3: planted bug is live and the decoy credential can't trip a // live-format scanner. const price = fs.readFileSync(path.join(FIXTURES, 'bugfix-decoys', 'src', 'format-price.js'), 'utf-8'); expect(price).toContain("'$' + dollars + '.' + rem"); const config = fs.readFileSync(path.join(FIXTURES, 'bugfix-decoys', 'src', 'config.js'), 'utf-8'); expect(config).toContain('not-a-real-credential'); expect(config).not.toMatch(/sk-[a-zA-Z0-9]{16,}/); // Decoy over-build invitations are planted. const readme = fs.readFileSync(path.join(FIXTURES, 'bugfix-decoys', 'README.md'), 'utf-8'); expect(readme).toContain('plugin architecture'); }); test('behavioral skill is an extraction (ladder + bounded closer), not a whole-file copy', () => { const skill = buildBehavioralSkill(); expect(skill).toContain(`name: ${SKILL_NAME}`); expect(skill).toContain('## Search Before Building'); expect(skill).toContain('first rung that holds'); expect(skill).toContain('## Voice'); expect(skill).toContain('**Bounded closer.**'); // Telemetry tail stripped: a hermetic child must not write to the // operator's real ~/.gstack. expect(skill).not.toContain('Eureka'); // Extraction proof: none of ship's workflow rode along. expect(skill).not.toContain('## Preamble (run first)'); expect(skill).not.toContain('Review Readiness'); expect(skill.length).toBeLessThan(8192); }); test('with-arm installs the skill + routing line; without-arm installs neither; both get git + bare origin', () => { const withArm = setupArm(TASKS[0], 'with-skill'); const withoutArm = setupArm(TASKS[0], 'without-skill'); try { const skillPath = path.join(withArm.dir, '.claude', 'skills', SKILL_NAME, 'SKILL.md'); expect(fs.existsSync(skillPath)).toBe(true); expect(fs.readFileSync(path.join(withArm.dir, 'CLAUDE.md'), 'utf-8')).toContain('## Skill routing'); expect(fs.existsSync(path.join(withoutArm.dir, '.claude'))).toBe(false); expect(fs.readFileSync(path.join(withoutArm.dir, 'CLAUDE.md'), 'utf-8')).not.toContain('Skill routing'); // Both arms: seeded commit + working bare origin (merge-base-style // commands must work inside the arm). for (const arm of [withArm, withoutArm]) { expect(run('git', ['rev-parse', 'HEAD'], arm.dir).trim()).toMatch(/^[0-9a-f]{40}$/); expect(run('git', ['remote', 'get-url', 'origin'], arm.dir).trim()).toBe(arm.originDir); expect(run('git', ['merge-base', 'origin/main', 'HEAD'], arm.dir).trim()).toMatch(/^[0-9a-f]{40}$/); } } finally { for (const arm of [withArm, withoutArm]) { fs.rmSync(arm.dir, { recursive: true, force: true }); fs.rmSync(arm.originDir, { recursive: true, force: true }); } } }); test('diff capture: stat parsing + a real zero-diff and non-zero-diff round trip', () => { expect(parseDiffStat(' 3 files changed, 120 insertions(+), 4 deletions(-)\n')) .toEqual({ filesChanged: 3, insertions: 120, deletions: 4, net: 116 }); expect(parseDiffStat(' 1 file changed, 2 insertions(+)\n')) .toEqual({ filesChanged: 1, insertions: 2, deletions: 0, net: 2 }); expect(parseDiffStat('')) .toEqual({ filesChanged: 0, insertions: 0, deletions: 0, net: 0 }); const arm = setupArm(TASKS[2], 'without-skill'); try { // Zero-diff arm: a VALID cell, zeros across the board. const clean = captureStagedDiff(arm.dir, arm.seedSha); expect(clean.filesChanged).toBe(0); expect(clean.net).toBe(0); expect(clean.patch.trim()).toBe(''); // Modify + add a file: counts appear, patch carries the change. fs.appendFileSync(path.join(arm.dir, 'README.md'), 'appended line\n'); fs.writeFileSync(path.join(arm.dir, 'new-file.txt'), 'one\ntwo\n'); const dirty = captureStagedDiff(arm.dir, arm.seedSha); expect(dirty.filesChanged).toBe(2); expect(dirty.insertions).toBe(3); expect(dirty.deletions).toBe(0); expect(dirty.net).toBe(3); expect(dirty.patch).toContain('appended line'); } finally { fs.rmSync(arm.dir, { recursive: true, force: true }); fs.rmSync(arm.originDir, { recursive: true, force: true }); } }); test('judge prompt construction embeds the rubric, the ticket, and the reference diffs', () => { const goodDiff = fs.readFileSync(path.join(FIXTURES, 'reference', 'good-diff.patch'), 'utf-8'); const badDiff = fs.readFileSync(path.join(FIXTURES, 'reference', 'bad-diff.patch'), 'utf-8'); for (const diff of [goodDiff, badDiff]) { const prompt = buildArmJudgePrompt(TASKS[0].ticket, diff, 'pinned0000'); expect(prompt).toContain('<<>>'); expect(prompt).toContain('<<>>'); expect(prompt).toContain(diff); expect(prompt).toContain(TASKS[0].ticket); expect(prompt).toContain('0-3 scale'); expect(prompt).toContain('Coverage is NOT over-engineering'); expect(prompt).toContain('MUST name the specific class, function, file, or pattern'); expect(prompt).toContain('construct MUST be exactly "none"'); } // The reference diffs are what the rubric anchors describe: the bad diff // carries a hand-rolled widget replacing a native element, the good one // uses the platform. expect(badDiff).toContain('class CalendarWidget'); expect(goodDiff).toContain('type="date"'); // Injection hardening: without an explicit sentinel, each call gets its // own random block markers — an arm diff cannot pre-write a closing // marker it has never seen. const a = buildArmJudgePrompt(TASKS[0].ticket, goodDiff); const b = buildArmJudgePrompt(TASKS[0].ticket, goodDiff); const marker = (p: string) => /<<>>/.exec(p)?.[1]; expect(marker(a)).toBeTruthy(); expect(marker(b)).toBeTruthy(); expect(marker(a)).not.toBe(marker(b)); }); test('judge response parsing: reference-shaped verdicts accepted, malformed rejected', () => { // Canned verdicts the judge should return for the reference diffs. const goodVerdict = parseArmJudgeResponse({ over_engineering: 0, construct: 'none', reasoning: 'Native date input with a min attribute; nothing unrequested.', }); expect(goodVerdict.over_engineering).toBe(0); expect(goodVerdict.construct).toBe('none'); const badVerdict = parseArmJudgeResponse({ over_engineering: 3, construct: 'hand-rolled CalendarWidget + DatePickerFactory in calendar.js', reasoning: 'A custom calendar widget layer replaces .', }); expect(badVerdict.over_engineering).toBe(3); expect(badVerdict.construct).toContain('CalendarWidget'); // Malformed shapes throw — that throw is what the bounded retry catches. expect(() => parseArmJudgeResponse({ over_engineering: 7, construct: 'x' })).toThrow(/integer 0-3/); expect(() => parseArmJudgeResponse({ over_engineering: 1.5, construct: 'x' })).toThrow(/integer 0-3/); expect(() => parseArmJudgeResponse({ over_engineering: 2 })).toThrow(/construct missing/); expect(() => parseArmJudgeResponse({ over_engineering: 2, construct: 'none' })).toThrow(/must name the specific construct/); expect(() => parseArmJudgeResponse({ over_engineering: 0, construct: 'a helper' })).toThrow(/construct "none"/); expect(() => parseArmJudgeResponse(null)).toThrow(); }); test('armJudge: zero diff scores deterministically as none with no API call', async () => { // No ANTHROPIC client is ever constructed on this path — safe keyless. const score = await armJudge(TASKS[0].ticket, ' \n'); expect(score.over_engineering).toBe(0); expect(score.construct).toBe('none'); }); test('armJudge: bounded retry-on-malformed — recovers once, then gives up', async () => { // Malformed first, valid second: recovers within the 2-attempt bound. let calls = 0; const flaky = (async () => { calls++; return calls === 1 ? { over_engineering: 9, construct: 'garbage' } : { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' }; }) as unknown as typeof callJudge; const recovered = await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: flaky }); expect(recovered.over_engineering).toBe(2); expect(calls).toBe(ARM_JUDGE_ATTEMPTS); // Always malformed: throws after exactly ARM_JUDGE_ATTEMPTS attempts. let badCalls = 0; const alwaysBad = (async () => { badCalls++; return { nonsense: true }; }) as unknown as typeof callJudge; await expect(armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: alwaysBad })) .rejects.toThrow(/no well-formed verdict after 2 attempts/); expect(badCalls).toBe(ARM_JUDGE_ATTEMPTS); }); }); describe('functional checks (correctness before LOC)', () => { test('every fixture with a run-tests.js oracle declares checkCmd; the trap fixtures behave as planted', () => { for (const task of TASKS) { const oracle = path.join(FIXTURES, task.fixture, 'run-tests.js'); if (fs.existsSync(oracle)) { expect(task.checkCmd, `${task.key} has run-tests.js but no checkCmd — its cells would report checks=none`).toEqual(['node', 'run-tests.js']); } else { expect(task.checkCmd).toBeUndefined(); } } // Pre-fix, the bugfix fixture MUST fail its own oracle (the planted bug), // and a task with no oracle reports 'none' — never a throw. const arm = setupArm(TASKS[2], 'without-skill'); const noOracle = setupArm(TASKS[0], 'without-skill'); try { expect(runChecks(TASKS[2], arm.dir)).toBe('fail'); expect(runChecks(TASKS[0], noOracle.dir)).toBe('none'); } finally { for (const a of [arm, noOracle]) { fs.rmSync(a.dir, { recursive: true, force: true }); fs.rmSync(a.originDir, { recursive: true, force: true }); } } }); });