mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-17 02:15:31 +02:00
feat(evals): arm benchmark runs each fixture's functional oracle — correctness before LOC
The plan's metric order is diff-quality FIRST, but cells never ran the fixtures' own run-tests.js, so a refusal, a broken implementation, and working code were indistinguishable in aggregates (Codex adversarial catch). Tasks with an oracle declare checkCmd; every cell records checks=pass|fail|none in the report line and eval store. Selftest pins the oracle declarations and that the planted bug fails its own check pre-fix. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d13f1e07c7
commit
c9e9a653cf
@@ -9,7 +9,7 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
TASKS, FIXTURES, SKILL_NAME,
|
||||
buildBehavioralSkill, run, setupArm, parseDiffStat, captureStagedDiff,
|
||||
buildBehavioralSkill, run, setupArm, parseDiffStat, captureStagedDiff, runChecks,
|
||||
} from './helpers/arm-benchmark-harness';
|
||||
import {
|
||||
armJudge, buildArmJudgePrompt, parseArmJudgeResponse,
|
||||
@@ -206,3 +206,30 @@ describe('arm benchmark selftest (free, no API)', () => {
|
||||
expect(badCalls).toBe(ARM_JUDGE_ATTEMPTS);
|
||||
});
|
||||
});
|
||||
|
||||
describe('functional checks (correctness before LOC)', () => {
|
||||
test('every fixture with a run-tests.js oracle declares checkCmd; the trap fixtures behave as planted', () => {
|
||||
for (const task of TASKS) {
|
||||
const oracle = path.join(FIXTURES, task.fixture, 'run-tests.js');
|
||||
if (fs.existsSync(oracle)) {
|
||||
expect(task.checkCmd, `${task.key} has run-tests.js but no checkCmd — its cells would report checks=none`).toEqual(['node', 'run-tests.js']);
|
||||
} else {
|
||||
expect(task.checkCmd).toBeUndefined();
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-fix, the bugfix fixture MUST fail its own oracle (the planted bug),
|
||||
// and a task with no oracle reports 'none' — never a throw.
|
||||
const arm = setupArm(TASKS[2], 'without-skill');
|
||||
const noOracle = setupArm(TASKS[0], 'without-skill');
|
||||
try {
|
||||
expect(runChecks(TASKS[2], arm.dir)).toBe('fail');
|
||||
expect(runChecks(TASKS[0], noOracle.dir)).toBe('none');
|
||||
} finally {
|
||||
for (const a of [arm, noOracle]) {
|
||||
fs.rmSync(a.dir, { recursive: true, force: true });
|
||||
fs.rmSync(a.originDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user