feat(evals): arm benchmark runs each fixture's functional oracle — correctness before LOC

The plan's metric order is diff-quality FIRST, but cells never ran the
fixtures' own run-tests.js, so a refusal, a broken implementation, and
working code were indistinguishable in aggregates (Codex adversarial catch).
Tasks with an oracle declare checkCmd; every cell records checks=pass|fail|none
in the report line and eval store. Selftest pins the oracle declarations and
that the planted bug fails its own check pre-fix.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:57:56 +00:00
co-authored by Claude Fable 5
parent d13f1e07c7
commit c9e9a653cf
3 changed files with 55 additions and 4 deletions
+28 -1
View File
@@ -9,7 +9,7 @@
import { describe, test, expect } from 'bun:test';
import {
TASKS, FIXTURES, SKILL_NAME,
buildBehavioralSkill, run, setupArm, parseDiffStat, captureStagedDiff,
buildBehavioralSkill, run, setupArm, parseDiffStat, captureStagedDiff, runChecks,
} from './helpers/arm-benchmark-harness';
import {
armJudge, buildArmJudgePrompt, parseArmJudgeResponse,
@@ -206,3 +206,30 @@ describe('arm benchmark selftest (free, no API)', () => {
expect(badCalls).toBe(ARM_JUDGE_ATTEMPTS);
});
});
describe('functional checks (correctness before LOC)', () => {
test('every fixture with a run-tests.js oracle declares checkCmd; the trap fixtures behave as planted', () => {
for (const task of TASKS) {
const oracle = path.join(FIXTURES, task.fixture, 'run-tests.js');
if (fs.existsSync(oracle)) {
expect(task.checkCmd, `${task.key} has run-tests.js but no checkCmd — its cells would report checks=none`).toEqual(['node', 'run-tests.js']);
} else {
expect(task.checkCmd).toBeUndefined();
}
}
// Pre-fix, the bugfix fixture MUST fail its own oracle (the planted bug),
// and a task with no oracle reports 'none' — never a throw.
const arm = setupArm(TASKS[2], 'without-skill');
const noOracle = setupArm(TASKS[0], 'without-skill');
try {
expect(runChecks(TASKS[2], arm.dir)).toBe('fail');
expect(runChecks(TASKS[0], noOracle.dir)).toBe('none');
} finally {
for (const a of [arm, noOracle]) {
fs.rmSync(a.dir, { recursive: true, force: true });
fs.rmSync(a.originDir, { recursive: true, force: true });
}
}
});
});