CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-14 08:59:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(evals): arm benchmark runs each fixture's functional oracle — correctness before LOC

Browse Source 
The plan's metric order is diff-quality FIRST, but cells never ran the
fixtures' own run-tests.js, so a refusal, a broken implementation, and
working code were indistinguishable in aggregates (Codex adversarial catch).
Tasks with an oracle declare checkCmd; every cell records checks=pass|fail|none
in the report line and eval store. Selftest pins the oracle declarations and
that the planted bug fails its own check pre-fix.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:57:56 +00:00
co-authored by Claude Fable 5
parent d13f1e07c7
commit c9e9a653cf
3 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-arm-benchmark.test.ts
+11 -3
View File
@@ -45,7 +45,7 @@ import { armJudge, type ArmJudgeScore } from './helpers/llm-judge';
import {
ARM_MAX_TURNS, ARM_TIMEOUT_MS, ARM_JUDGE_DIFF_CAP, ARM_ALLOWED_TOOLS,
TASK_TEST_TIMEOUT_MS, SKILL_NAME, TASKS,
setupArm, captureStagedDiff,
setupArm, captureStagedDiff, runChecks,
type Arm, type ArmTask, type DiffHarvest,
} from './helpers/arm-benchmark-harness';
import * as fs from 'fs';
@@ -61,6 +61,10 @@ interface CellResult {
harvestError: string | null;
judge: ArmJudgeScore | null;
judgeError: string | null;
/** Deterministic functional-check outcome ('none' = task has no oracle).
* Correctness comes before LOC in the metric order — a refusal, a broken
* implementation, and working code must be distinguishable in the cells. */
checks: 'pass' | 'fail' | 'none';
consulted: boolean;
costUsd: number;
tokens: number;
@@ -102,6 +106,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
} catch (err) {
harvestError = err instanceof Error ? err.message : String(err);
}
const checks = runChecks(task, dirs.dir);
// Judge taxonomy: still malformed after armJudge's bounded retries ->
// judge_error cell (excluded from aggregates, surfaced in the report).
@@ -131,7 +136,9 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
net: harvest.net,
}
: null,
judge_scores: judge ? { over_engineering: judge.over_engineering } : undefined,
judge_scores: judge
? { over_engineering: judge.over_engineering, ...(checks !== 'none' ? { checks_pass: checks === 'pass' ? 1 : 0 } : {}) }
: undefined,
judge_reasoning: judge
? `construct: ${judge.construct} | ${judge.reasoning}${judgeDiffTruncated ? ` | diff truncated to ${ARM_JUDGE_DIFF_CAP}B` : ''}`
: judgeError ? `judge_error: ${judgeError}` : undefined,
@@ -146,6 +153,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
harvestError,
judge,
judgeError,
checks,
consulted,
costUsd: result.costEstimate.estimatedCost,
tokens: result.costEstimate.estimatedTokens,
@@ -166,7 +174,7 @@ function cellLine(c: CellResult): string {
const loc = c.harvest
? `+${c.harvest.insertions}/-${c.harvest.deletions} net ${c.harvest.net} in ${c.harvest.filesChanged} file(s)`
: `harvest FAILED: ${c.harvestError}`;
return ` ${c.arm.padEnd(14)} score=${score} loc=${loc} turns=${c.turns} `
return ` ${c.arm.padEnd(14)} score=${score} checks=${c.checks} loc=${loc} turns=${c.turns} `
+ `tokens=${(c.tokens / 1000).toFixed(1)}k cost=$${c.costUsd.toFixed(2)} consulted=${c.consulted}`;
}