mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-10 06:58:59 +02:00
refactor(evals): arm-benchmark selftest runs FREE on every PR
The selftest lived inside the paid skill-e2e-* file, so fixture-integrity and plumbing pins executed weekly at best — a broken fixture would ship past every gating check and be discovered when the periodic run burned money on a dead instrument. Harness extracted to test/helpers/arm-benchmark-harness.ts, selftest to test/arm-benchmark-selftest.test.ts (free suite). Touchfiles: harness added to the three benchmark dep lists; the auq-repetition-cut-ab tier comment now states the MANUAL re-run obligation honestly (periodic runs force EVALS_ALL, so dep lists cannot auto-trigger it). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
760554d0ac
commit
2ee61ab455
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Arm-benchmark harness — shared by the paid periodic benchmark
|
||||
* (test/skill-e2e-arm-benchmark.test.ts) and the FREE selftest
|
||||
* (test/arm-benchmark-selftest.test.ts). Extracted so the fixture-integrity
|
||||
* and plumbing pins run in `bun run test` on every PR: the paid file matches
|
||||
* the skill-e2e-* paid glob, so a selftest living inside it executed weekly
|
||||
* at best — a broken fixture would ship past every gating check and be
|
||||
* discovered only when the periodic run burned money on a dead instrument.
|
||||
*/
|
||||
import { ROOT, copyDirSync } from './e2e-helpers';
|
||||
import { extractSkillSections } from './skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
// --- Named per-arm constants (plan: defaults of 15 turns/120s are nowhere
|
||||
// near enough for a build-shaped ticket: read fixture, implement, run tests).
|
||||
export const ARM_MAX_TURNS = 40;
|
||||
export const ARM_TIMEOUT_MS = 8 * 60_000;
|
||||
/** Max diff bytes sent to the over-engineering judge. Truncation is loud
|
||||
* (logged + suffixed onto judge_reasoning) — a clipped patch can hide the
|
||||
* construct being scored, so a silent cap would corrupt cells invisibly. */
|
||||
export const ARM_JUDGE_DIFF_CAP = 30_000;
|
||||
// Skill tool in BOTH arms so the tool surface is symmetric — the without-arm
|
||||
// simply has nothing installed to invoke. No Agent: build-discipline
|
||||
// dispatches no subagents.
|
||||
export const ARM_ALLOWED_TOOLS = ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'Skill'];
|
||||
// Two concurrent arms + up to two judge calls + fixture setup.
|
||||
export const TASK_TEST_TIMEOUT_MS = ARM_TIMEOUT_MS + 240_000;
|
||||
|
||||
export const SKILL_NAME = 'build-discipline';
|
||||
export const FIXTURES = path.join(ROOT, 'test', 'fixtures', 'arm-benchmark');
|
||||
|
||||
export type Arm = 'with-skill' | 'without-skill';
|
||||
|
||||
export interface ArmTask {
|
||||
/** E2E_TOUCHFILES / E2E_TIERS key. Quoted literals below keep the parent
|
||||
* shard mapper (test-paid-shards) able to attribute this file. */
|
||||
key: string;
|
||||
fixture: string;
|
||||
ticket: string;
|
||||
}
|
||||
|
||||
// The 3 named tasks (plan: one native-platform over-build trap, one CRUD
|
||||
// endpoint, one bug-fix with planted decoy over-build invitations).
|
||||
export const TASKS: ArmTask[] = [
|
||||
{
|
||||
key: 'arm-benchmark-native-overbuild',
|
||||
fixture: 'native-overbuild',
|
||||
ticket: `Ticket: hikers need to pick their hike date when booking.
|
||||
|
||||
You are in a small git repo containing a static site (index.html, app.js, styles.css — no build step).
|
||||
Add a date field to the booking form: it must be required, must not allow choosing a past date, and the confirmation message must include the chosen date.
|
||||
|
||||
Leave your changes uncommitted in the working tree.`,
|
||||
},
|
||||
{
|
||||
key: 'arm-benchmark-crud-endpoint',
|
||||
fixture: 'crud-endpoint',
|
||||
ticket: `Ticket: users need to delete notes.
|
||||
|
||||
You are in a small git repo containing an in-memory notes API (app.js, wired to HTTP in server.js).
|
||||
Add DELETE /notes/:id: respond 204 on success and 404 for an unknown id, and cover the new endpoint in run-tests.js. Verify with: node run-tests.js
|
||||
|
||||
Leave your changes uncommitted in the working tree.`,
|
||||
},
|
||||
{
|
||||
key: 'arm-benchmark-bugfix-decoys',
|
||||
fixture: 'bugfix-decoys',
|
||||
ticket: `Bug report: receipts print $10.5 for a $10.05 item.
|
||||
|
||||
You are in a small git repo. \`node run-tests.js\` currently fails on formatPrice(1005).
|
||||
Fix the bug so all tests pass. Verify with: node run-tests.js
|
||||
|
||||
Leave your changes uncommitted in the working tree.`,
|
||||
},
|
||||
];
|
||||
|
||||
// --- Skill under test: extracted behavioral layer ---
|
||||
|
||||
/** Drop the Eureka telemetry tail from the extracted Search Before Building
|
||||
* section: it appends to the OPERATOR's real ~/.gstack from inside a
|
||||
* hermetic child, and telemetry is not the behavior under test. */
|
||||
export function stripEureka(text: string): string {
|
||||
const start = text.indexOf('**Eureka:**');
|
||||
if (start === -1) return text;
|
||||
const next = text.indexOf('\n## ', start);
|
||||
return text.slice(0, start) + (next === -1 ? '' : text.slice(next + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the behavioral-layer skill: the WS3 reuse ladder (## Search Before
|
||||
* Building) + the WS7 bounded closer (## Voice), extracted from the rendered
|
||||
* ship/SKILL.md (tier 4 — carries both sections) and wrapped in this
|
||||
* benchmark's own frontmatter. Extract, don't copy (CLAUDE.md rule).
|
||||
*/
|
||||
export function buildBehavioralSkill(): string {
|
||||
const extracted = extractSkillSections(path.join(ROOT, 'ship'), ['Search Before Building', 'Voice']);
|
||||
const body = stripEureka(extracted.replace(/^---\n[\s\S]*?\n---\n/, '')).trim();
|
||||
return `---
|
||||
name: ${SKILL_NAME}
|
||||
description: Build discipline for implementation tickets — the reuse ladder (stop at the first rung that holds) plus bounded completion reports. Invoke before implementing any ticket.
|
||||
---
|
||||
|
||||
# Build discipline
|
||||
|
||||
Apply these rules to the implementation work you are about to do.
|
||||
|
||||
${body}
|
||||
`;
|
||||
}
|
||||
|
||||
// --- Arm setup: fixture copy + optional skill install + git init + bare origin ---
|
||||
|
||||
export interface ArmDirs {
|
||||
dir: string;
|
||||
originDir: string;
|
||||
/** The seed commit — the immutable diff base for harvest (an agent that
|
||||
* disobeys "leave uncommitted" by committing AND pushing can move
|
||||
* origin/main, but it cannot move a recorded SHA). */
|
||||
seedSha: string;
|
||||
}
|
||||
|
||||
export function run(cmd: string, args: string[], cwd: string): string {
|
||||
// 64MB maxBuffer: the patch capture pipes the FULL staged diff through
|
||||
// here, and the most over-built arm outcome (vendored dependency) is
|
||||
// exactly the one the benchmark must not die on.
|
||||
const r = spawnSync(cmd, args, { cwd, stdio: 'pipe', encoding: 'utf-8', timeout: 15_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (r.status !== 0) {
|
||||
throw new Error(`${cmd} ${args.join(' ')} failed in ${cwd}: ${r.stderr || r.stdout}`);
|
||||
}
|
||||
return r.stdout ?? '';
|
||||
}
|
||||
|
||||
export function setupArm(task: ArmTask, arm: Arm): ArmDirs {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), `arm-${task.fixture}-${arm}-`));
|
||||
copyDirSync(path.join(FIXTURES, task.fixture), dir);
|
||||
|
||||
const baseClaudeMd = '# Project\n\nSmall fixture repo for an implementation ticket. Run its checks with the command named in the ticket.\n';
|
||||
if (arm === 'with-skill') {
|
||||
const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME);
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill());
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'CLAUDE.md'),
|
||||
baseClaudeMd
|
||||
+ `\n## Skill routing\n\nBefore implementing any ticket, invoke the ${SKILL_NAME} skill via the Skill tool and follow it while you work.\n`,
|
||||
);
|
||||
} else {
|
||||
fs.writeFileSync(path.join(dir, 'CLAUDE.md'), baseClaudeMd);
|
||||
}
|
||||
|
||||
// node_modules never enters the harvest: an arm that npm-installs a
|
||||
// dependency is a scoreable outcome, not a reason to stage 10k files.
|
||||
if (!fs.existsSync(path.join(dir, '.gitignore'))) {
|
||||
fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\n');
|
||||
}
|
||||
|
||||
run('git', ['init', '-b', 'main'], dir);
|
||||
run('git', ['config', 'user.email', 'arm-bench@example.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Arm Bench'], dir);
|
||||
run('git', ['config', 'commit.gpgsign', 'false'], dir);
|
||||
run('git', ['add', '-A'], dir);
|
||||
run('git', ['commit', '-m', 'seed fixture'], dir);
|
||||
const seedSha = run('git', ['rev-parse', 'HEAD'], dir).trim();
|
||||
|
||||
// Local bare origin so merge-base-style commands work inside the arm.
|
||||
const originDir = fs.mkdtempSync(path.join(os.tmpdir(), `arm-${task.fixture}-${arm}-origin-`));
|
||||
run('git', ['init', '--bare', '-b', 'main'], originDir);
|
||||
run('git', ['remote', 'add', 'origin', originDir], dir);
|
||||
run('git', ['push', '-u', 'origin', 'main'], dir);
|
||||
|
||||
return { dir, originDir, seedSha };
|
||||
}
|
||||
|
||||
// --- Diff capture: git add -A && git diff --cached --stat (plan spec) ---
|
||||
|
||||
export interface DiffHarvest {
|
||||
filesChanged: number;
|
||||
insertions: number;
|
||||
deletions: number;
|
||||
net: number;
|
||||
stat: string;
|
||||
patch: string;
|
||||
}
|
||||
|
||||
/** Parse the summary line of `git diff --stat`. Empty stat = zero-diff
|
||||
* (a VALID cell, not an error). */
|
||||
export function parseDiffStat(stat: string): Pick<DiffHarvest, 'filesChanged' | 'insertions' | 'deletions' | 'net'> {
|
||||
const line = stat.trim().split('\n').pop() ?? '';
|
||||
const files = line.match(/(\d+) files? changed/);
|
||||
const ins = line.match(/(\d+) insertions?\(\+\)/);
|
||||
const del = line.match(/(\d+) deletions?\(-\)/);
|
||||
const insertions = ins ? Number(ins[1]) : 0;
|
||||
const deletions = del ? Number(del[1]) : 0;
|
||||
return {
|
||||
filesChanged: files ? Number(files[1]) : 0,
|
||||
insertions,
|
||||
deletions,
|
||||
net: insertions - deletions,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rung 2: three lines of git beat a generalized manager (WorktreeManager only
|
||||
* harvests worktrees it created from the gstack repo — it cannot harvest
|
||||
* synthetic fixtures). Diffing the index against the RECORDED seed SHA (not
|
||||
* origin/main, which an agent that commits AND pushes can move; not HEAD,
|
||||
* which a plain commit moves) keeps the capture honest under every flavor of
|
||||
* "leave uncommitted" disobedience.
|
||||
*/
|
||||
export function captureStagedDiff(dir: string, seedSha: string): DiffHarvest {
|
||||
run('git', ['add', '-A'], dir);
|
||||
const stat = run('git', ['diff', '--cached', seedSha, '--stat'], dir);
|
||||
const patch = run('git', ['diff', '--cached', seedSha], dir);
|
||||
return { ...parseDiffStat(stat), stat: stat.trim(), patch };
|
||||
}
|
||||
|
||||
@@ -454,6 +454,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'scripts/resolvers/preamble/generate-voice-directive.ts',
|
||||
'test/fixtures/arm-benchmark/**',
|
||||
'test/helpers/llm-judge.ts',
|
||||
'test/helpers/arm-benchmark-harness.ts',
|
||||
'test/skill-e2e-arm-benchmark.test.ts',
|
||||
'ship/SKILL.md',
|
||||
],
|
||||
@@ -462,6 +463,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'scripts/resolvers/preamble/generate-voice-directive.ts',
|
||||
'test/fixtures/arm-benchmark/**',
|
||||
'test/helpers/llm-judge.ts',
|
||||
'test/helpers/arm-benchmark-harness.ts',
|
||||
'test/skill-e2e-arm-benchmark.test.ts',
|
||||
'ship/SKILL.md',
|
||||
],
|
||||
@@ -470,6 +472,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'scripts/resolvers/preamble/generate-voice-directive.ts',
|
||||
'test/fixtures/arm-benchmark/**',
|
||||
'test/helpers/llm-judge.ts',
|
||||
'test/helpers/arm-benchmark-harness.ts',
|
||||
'test/skill-e2e-arm-benchmark.test.ts',
|
||||
'ship/SKILL.md',
|
||||
],
|
||||
@@ -581,7 +584,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
// periodic: long-running or expensive (>$3/run), run weekly
|
||||
'preamble-script-ab': 'periodic', // Phase 1-3 A/B: script vs inline preamble; demoted post-Phase-3 (OV7)
|
||||
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing; re-runs on AUQ format changes)
|
||||
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing). Periodic runs force EVALS_ALL, so the dep list cannot auto-trigger it — an AUQ format edit carries a MANUAL re-run obligation (bun test test/skill-e2e-auq-repetition-cut-ab.test.ts with EVALS=1 EVALS_TIER=periodic)
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
|
||||
Reference in New Issue
Block a user