gstack/test/helpers/arm-benchmark-harness.ts

/**
 * Arm-benchmark harness — shared by the paid periodic benchmark
 * (test/skill-e2e-arm-benchmark.test.ts) and the FREE selftest
 * (test/arm-benchmark-selftest.test.ts). Extracted so the fixture-integrity
 * and plumbing pins run in `bun run test` on every PR: the paid file matches
 * the skill-e2e-* paid glob, so a selftest living inside it executed weekly
 * at best — a broken fixture would ship past every gating check and be
 * discovered only when the periodic run burned money on a dead instrument.
 */
import { ROOT, copyDirSync } from './e2e-helpers';
import { extractSkillSections } from './skill-fixture';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';

// --- Named per-arm constants (plan: defaults of 15 turns/120s are nowhere
// near enough for a build-shaped ticket: read fixture, implement, run tests).
export const ARM_MAX_TURNS = 40;
export const ARM_TIMEOUT_MS = 8 * 60_000;
/** Max diff bytes sent to the over-engineering judge. Truncation is loud
 *  (logged + suffixed onto judge_reasoning) — a clipped patch can hide the
 *  construct being scored, so a silent cap would corrupt cells invisibly. */
export const ARM_JUDGE_DIFF_CAP = 30_000;
// Skill tool in BOTH arms so the tool surface is symmetric — the without-arm
// simply has nothing installed to invoke. No Agent: build-discipline
// dispatches no subagents.
export const ARM_ALLOWED_TOOLS = ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'Skill'];
// Two concurrent arms + up to two judge calls + fixture setup.
export const TASK_TEST_TIMEOUT_MS = ARM_TIMEOUT_MS + 240_000;

export const SKILL_NAME = 'build-discipline';
export const FIXTURES = path.join(ROOT, 'test', 'fixtures', 'arm-benchmark');

export type Arm = 'with-skill' | 'without-skill';

export interface ArmTask {
  /** E2E_TOUCHFILES / E2E_TIERS key. Quoted literals below keep the parent
   *  shard mapper (test-paid-shards) able to attribute this file. */
  key: string;
  /** Deterministic functional check run against the arm's working tree after
   *  harvest (the plan's metric order is diff-quality FIRST: correctness
   *  before LOC). Absent for fixtures with no runnable oracle. */
  checkCmd?: string[];
  fixture: string;
  ticket: string;
}

// The 3 named tasks (plan: one native-platform over-build trap, one CRUD
// endpoint, one bug-fix with planted decoy over-build invitations).
export const TASKS: ArmTask[] = [
  {
    key: 'arm-benchmark-native-overbuild',
    fixture: 'native-overbuild',
    ticket: `Ticket: hikers need to pick their hike date when booking.

You are in a small git repo containing a static site (index.html, app.js, styles.css — no build step).
Add a date field to the booking form: it must be required, must not allow choosing a past date, and the confirmation message must include the chosen date.

Leave your changes uncommitted in the working tree.`,
  },
  {
    key: 'arm-benchmark-crud-endpoint',
    fixture: 'crud-endpoint',
    checkCmd: ['node', 'run-tests.js'],
    ticket: `Ticket: users need to delete notes.

You are in a small git repo containing an in-memory notes API (app.js, wired to HTTP in server.js).
Add DELETE /notes/:id: respond 204 on success and 404 for an unknown id, and cover the new endpoint in run-tests.js. Verify with: node run-tests.js

Leave your changes uncommitted in the working tree.`,
  },
  {
    key: 'arm-benchmark-bugfix-decoys',
    fixture: 'bugfix-decoys',
    checkCmd: ['node', 'run-tests.js'],
    ticket: `Bug report: receipts print $10.5 for a $10.05 item.

You are in a small git repo. \`node run-tests.js\` currently fails on formatPrice(1005).
Fix the bug so all tests pass. Verify with: node run-tests.js

Leave your changes uncommitted in the working tree.`,
  },
];

// --- Skill under test: extracted behavioral layer ---

/** Drop the Eureka telemetry tail from the extracted Search Before Building
 *  section: it appends to the OPERATOR's real ~/.gstack from inside a
 *  hermetic child, and telemetry is not the behavior under test. */
export function stripEureka(text: string): string {
  const start = text.indexOf('**Eureka:**');
  if (start === -1) return text;
  const next = text.indexOf('\n## ', start);
  return text.slice(0, start) + (next === -1 ? '' : text.slice(next + 1));
}

/**
 * Assemble the behavioral-layer skill: the WS3 reuse ladder (## Search Before
 * Building) + the WS7 bounded closer (## Voice), extracted from the rendered
 * ship/SKILL.md (tier 4 — carries both sections) and wrapped in this
 * benchmark's own frontmatter. Extract, don't copy (CLAUDE.md rule).
 */
export function buildBehavioralSkill(): string {
  const extracted = extractSkillSections(path.join(ROOT, 'ship'), ['Search Before Building', 'Voice']);
  const body = stripEureka(extracted.replace(/^---\n[\s\S]*?\n---\n/, '')).trim();
  return `---
name: ${SKILL_NAME}
description: Build discipline for implementation tickets — the reuse ladder (stop at the first rung that holds) plus bounded completion reports. Invoke before implementing any ticket.
---

# Build discipline

Apply these rules to the implementation work you are about to do.

${body}
`;
}

// --- Arm setup: fixture copy + optional skill install + git init + bare origin ---

export interface ArmDirs {
  dir: string;
  originDir: string;
  /** The seed commit — the immutable diff base for harvest (an agent that
   *  disobeys "leave uncommitted" by committing AND pushing can move
   *  origin/main, but it cannot move a recorded SHA). */
  seedSha: string;
}

export function run(cmd: string, args: string[], cwd: string): string {
  // 64MB maxBuffer: the patch capture pipes the FULL staged diff through
  // here, and the most over-built arm outcome (vendored dependency) is
  // exactly the one the benchmark must not die on.
  const r = spawnSync(cmd, args, { cwd, stdio: 'pipe', encoding: 'utf-8', timeout: 15_000, maxBuffer: 64 * 1024 * 1024 });
  if (r.status !== 0) {
    throw new Error(`${cmd} ${args.join(' ')} failed in ${cwd}: ${r.stderr || r.stdout}`);
  }
  return r.stdout ?? '';
}

export function setupArm(task: ArmTask, arm: Arm): ArmDirs {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), `arm-${task.fixture}-${arm}-`));
  copyDirSync(path.join(FIXTURES, task.fixture), dir);

  const baseClaudeMd = '# Project\n\nSmall fixture repo for an implementation ticket. Run its checks with the command named in the ticket.\n';
  if (arm === 'with-skill') {
    const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME);
    fs.mkdirSync(skillDir, { recursive: true });
    fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill());
    fs.writeFileSync(
      path.join(dir, 'CLAUDE.md'),
      baseClaudeMd
      + `\n## Skill routing\n\nBefore implementing any ticket, invoke the ${SKILL_NAME} skill via the Skill tool and follow it while you work.\n`,
    );
  } else {
    fs.writeFileSync(path.join(dir, 'CLAUDE.md'), baseClaudeMd);
  }

  // node_modules never enters the harvest: an arm that npm-installs a
  // dependency is a scoreable outcome, not a reason to stage 10k files.
  if (!fs.existsSync(path.join(dir, '.gitignore'))) {
    fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\n');
  }

  run('git', ['init', '-b', 'main'], dir);
  run('git', ['config', 'user.email', 'arm-bench@example.com'], dir);
  run('git', ['config', 'user.name', 'Arm Bench'], dir);
  run('git', ['config', 'commit.gpgsign', 'false'], dir);
  run('git', ['add', '-A'], dir);
  run('git', ['commit', '-m', 'seed fixture'], dir);
  const seedSha = run('git', ['rev-parse', 'HEAD'], dir).trim();

  // Local bare origin so merge-base-style commands work inside the arm.
  const originDir = fs.mkdtempSync(path.join(os.tmpdir(), `arm-${task.fixture}-${arm}-origin-`));
  run('git', ['init', '--bare', '-b', 'main'], originDir);
  run('git', ['remote', 'add', 'origin', originDir], dir);
  run('git', ['push', '-u', 'origin', 'main'], dir);

  return { dir, originDir, seedSha };
}

// --- Diff capture: git add -A && git diff --cached --stat (plan spec) ---

export interface DiffHarvest {
  filesChanged: number;
  insertions: number;
  deletions: number;
  net: number;
  stat: string;
  patch: string;
}

/** Parse the summary line of `git diff --stat`. Empty stat = zero-diff
 *  (a VALID cell, not an error). */
export function parseDiffStat(stat: string): Pick<DiffHarvest, 'filesChanged' | 'insertions' | 'deletions' | 'net'> {
  const line = stat.trim().split('\n').pop() ?? '';
  const files = line.match(/(\d+) files? changed/);
  const ins = line.match(/(\d+) insertions?\(\+\)/);
  const del = line.match(/(\d+) deletions?\(-\)/);
  const insertions = ins ? Number(ins[1]) : 0;
  const deletions = del ? Number(del[1]) : 0;
  return {
    filesChanged: files ? Number(files[1]) : 0,
    insertions,
    deletions,
    net: insertions - deletions,
  };
}

/**
 * Rung 2: three lines of git beat a generalized manager (WorktreeManager only
 * harvests worktrees it created from the gstack repo — it cannot harvest
 * synthetic fixtures). Diffing the index against the RECORDED seed SHA (not
 * origin/main, which an agent that commits AND pushes can move; not HEAD,
 * which a plain commit moves) keeps the capture honest under every flavor of
 * "leave uncommitted" disobedience.
 */
export function captureStagedDiff(dir: string, seedSha: string): DiffHarvest {
  run('git', ['add', '-A'], dir);
  const stat = run('git', ['diff', '--cached', seedSha, '--stat'], dir);
  const patch = run('git', ['diff', '--cached', seedSha], dir);
  return { ...parseDiffStat(stat), stat: stat.trim(), patch };
}


/** Run the task's functional check in the arm dir. 'none' when the task has
 *  no oracle; never throws — a crashing check is a 'fail', not a dead cell. */
export function runChecks(task: ArmTask, dir: string): 'pass' | 'fail' | 'none' {
  if (!task.checkCmd || task.checkCmd.length === 0) return 'none';
  const r = spawnSync(task.checkCmd[0], task.checkCmd.slice(1), {
    cwd: dir, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000,
  });
  return r.status === 0 ? 'pass' : 'fail';
}