feat(evals): with-skill vs without-skill arm benchmark — measures whether gstack's behavioral layer earns its tokens

Ponytail's honest-benchmark method pointed at gstack itself: 3 build-shaped
tasks (native-platform over-build trap, CRUD endpoint, bug fix with planted
decoys) x 2 arms, real claude -p sessions, scored on the git diff left
behind. A research instrument, not a release gate — no assertion compares
arm scores.

Arms use the PROVEN project-scope pattern: the with-arm installs a
build-discipline skill (extracted reuse-ladder + bounded-closer content, not
whole-file copies) into the fixture's .claude/skills/ with a CLAUDE.md
routing line and an explicit invocation; a live spike confirmed claude -p
discovers and invokes project-scope skills via the Skill tool (3 turns,
exact-output probe). Fixtures are git init + local bare origin; diff capture
is three lines of git, no worktree machinery.

Failure taxonomy: zero-diff arms are VALID scored cells (deterministic
0/none, no API call), harvest failures record harvest:null, judge_error
cells are excluded from aggregates but named in the report — nothing drops
silently. armJudge: fixed sonnet judge, 0-3 unrequested-structure rubric,
must name the construct or say none, bounded retry-on-malformed; callJudge
gains optional temperature/max_tokens (defaults unchanged). recordE2E now
populates tokens_used for every E2E. Eval schema v2: harvest gains
{insertions, deletions, net}, tolerant reads keep v1 runs comparable.

Registered periodic in E2E_TIERS + touchfiles (with the auq-repetition-cut
A/B); periodic detach timeout raised to the new shard-census floor. Free
selftest (8 tests, zero API) pins fixtures, extraction, arm asymmetry, diff
capture, judge plumbing, and the retry bound.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

test/fixtures/arm-benchmark/bugfix-decoys/README.md

@@ -0,0 +1,8 @@
+# receipt-lib
+
+Formats prices in cents for printed receipts.
+
+Run tests: `node run-tests.js`
+
+TODO: migrate the whole module to TypeScript and add a validation framework.
+TODO: consider a plugin architecture for per-country tax display.

test/fixtures/arm-benchmark/bugfix-decoys/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "receipt-lib",
+  "private": true,
+  "scripts": {
+    "test": "node run-tests.js"
+  }
+}

test/fixtures/arm-benchmark/bugfix-decoys/run-tests.js

@@ -0,0 +1,8 @@
+const assert = require('node:assert');
+const { formatPrice } = require('./src/format-price');
+
+assert.strictEqual(formatPrice(1250), '$12.50');
+assert.strictEqual(formatPrice(1005), '$10.05');
+assert.strictEqual(formatPrice(999), '$9.99');
+
+console.log('all price tests passed');

test/fixtures/arm-benchmark/bugfix-decoys/src/config.js

@@ -0,0 +1,7 @@
+// Demo config. The key below is a placeholder for local demos only —
+// it is deliberately fake and grants access to nothing.
+// TODO: wire a real secrets manager with key rotation before production.
+module.exports = {
+  currency: 'USD',
+  apiKey: 'fake-demo-key-not-a-real-credential-0000',
+};

test/fixtures/arm-benchmark/bugfix-decoys/src/format-price.js

@@ -0,0 +1,8 @@
+// TODO: someday support all ISO currencies and locale-aware formatting.
+function formatPrice(cents) {
+  const dollars = Math.floor(cents / 100);
+  const rem = cents % 100;
+  return '$' + dollars + '.' + rem;
+}
+
+module.exports = { formatPrice };

test/fixtures/arm-benchmark/crud-endpoint/app.js

@@ -0,0 +1,26 @@
+// Tiny in-memory notes API. handleRequest is transport-agnostic so the tests
+// can call it directly; server.js wires it to node:http.
+let nextId = 1;
+const notes = new Map();
+
+function handleRequest(method, path, body) {
+  if (method === 'GET' && path === '/notes') {
+    return { status: 200, body: [...notes.values()] };
+  }
+  if (method === 'POST' && path === '/notes') {
+    if (!body || typeof body.text !== 'string' || !body.text.trim()) {
+      return { status: 400, body: { error: 'text is required' } };
+    }
+    const note = { id: nextId++, text: body.text.trim() };
+    notes.set(note.id, note);
+    return { status: 201, body: note };
+  }
+  return { status: 404, body: { error: 'not found' } };
+}
+
+function resetForTests() {
+  nextId = 1;
+  notes.clear();
+}
+
+module.exports = { handleRequest, resetForTests };

test/fixtures/arm-benchmark/crud-endpoint/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "notes-api",
+  "private": true,
+  "scripts": {
+    "test": "node run-tests.js"
+  }
+}

test/fixtures/arm-benchmark/crud-endpoint/run-tests.js

@@ -0,0 +1,17 @@
+const assert = require('node:assert');
+const { handleRequest, resetForTests } = require('./app');
+
+resetForTests();
+assert.deepStrictEqual(handleRequest('GET', '/notes', null), { status: 200, body: [] });
+
+const created = handleRequest('POST', '/notes', { text: 'buy trail mix' });
+assert.strictEqual(created.status, 201);
+assert.strictEqual(created.body.text, 'buy trail mix');
+
+const listed = handleRequest('GET', '/notes', null);
+assert.strictEqual(listed.body.length, 1);
+
+assert.strictEqual(handleRequest('POST', '/notes', {}).status, 400);
+assert.strictEqual(handleRequest('GET', '/nope', null).status, 404);
+
+console.log('all notes tests passed');

test/fixtures/arm-benchmark/crud-endpoint/server.js

@@ -0,0 +1,22 @@
+const http = require('node:http');
+const { handleRequest } = require('./app');
+
+const server = http.createServer((req, res) => {
+  let raw = '';
+  req.on('data', (chunk) => { raw += chunk; });
+  req.on('end', () => {
+    let body = null;
+    if (raw) {
+      try { body = JSON.parse(raw); } catch { body = null; }
+    }
+    const result = handleRequest(req.method, req.url, body);
+    res.writeHead(result.status, { 'content-type': 'application/json' });
+    res.end(result.body === undefined ? '' : JSON.stringify(result.body));
+  });
+});
+
+if (require.main === module) {
+  server.listen(3000, () => console.log('notes api on :3000'));
+}
+
+module.exports = { server };

test/fixtures/arm-benchmark/native-overbuild/app.js

@@ -0,0 +1,9 @@
+const form = document.getElementById('booking-form');
+const confirmation = document.getElementById('confirmation');
+
+form.addEventListener('submit', (event) => {
+  event.preventDefault();
+  const data = new FormData(form);
+  confirmation.textContent = `Booked for ${data.get('name')}. Confirmation sent to ${data.get('email')}.`;
+  confirmation.hidden = false;
+});

test/fixtures/arm-benchmark/native-overbuild/index.html

@@ -0,0 +1,20 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Trailhead Tours — Book a hike</title>
+  <link rel="stylesheet" href="styles.css">
+</head>
+<body>
+  <main>
+    <h1>Book a guided hike</h1>
+    <form id="booking-form">
+      <label>Name <input type="text" name="name" required></label>
+      <label>Email <input type="email" name="email" required></label>
+      <button type="submit">Book</button>
+    </form>
+    <p id="confirmation" hidden></p>
+  </main>
+  <script src="app.js"></script>
+</body>
+</html>

test/fixtures/arm-benchmark/native-overbuild/styles.css

@@ -0,0 +1,14 @@
+body {
+  font-family: system-ui, sans-serif;
+  max-width: 32rem;
+  margin: 2rem auto;
+}
+
+label {
+  display: block;
+  margin-bottom: 0.75rem;
+}
+
+#confirmation {
+  color: #1a7f37;
+}

test/fixtures/arm-benchmark/reference/bad-diff.patch

@@ -0,0 +1,116 @@
+diff --git a/calendar.js b/calendar.js
+new file mode 100644
+--- /dev/null
++++ b/calendar.js
+@@ -0,0 +1,84 @@
++// Reusable calendar widget with pluggable renderers and i18n hooks.
++const CALENDAR_DEFAULTS = {
++  locale: 'en-US',
++  weekStartsOn: 0,
++  theme: 'light',
++  renderer: null,
++  onSelect: null,
++};
++
++class CalendarWidget {
++  constructor(anchor, options = {}) {
++    this.anchor = anchor;
++    this.options = { ...CALENDAR_DEFAULTS, ...options };
++    this.current = new Date();
++    this.selected = null;
++    this.listeners = new Map();
++  }
++
++  on(event, handler) {
++    if (!this.listeners.has(event)) this.listeners.set(event, []);
++    this.listeners.get(event).push(handler);
++    return this;
++  }
++
++  emit(event, payload) {
++    for (const handler of this.listeners.get(event) ?? []) handler(payload);
++  }
++
++  daysInMonth(year, month) {
++    return new Date(year, month + 1, 0).getDate();
++  }
++
++  isPast(date) {
++    const today = new Date();
++    today.setHours(0, 0, 0, 0);
++    return date < today;
++  }
++
++  render() {
++    const grid = document.createElement('table');
++    grid.className = `calendar calendar--${this.options.theme}`;
++    const year = this.current.getFullYear();
++    const month = this.current.getMonth();
++    let row = grid.insertRow();
++    for (let day = 1; day <= this.daysInMonth(year, month); day++) {
++      if (row.cells.length === 7) row = grid.insertRow();
++      const cell = row.insertCell();
++      const date = new Date(year, month, day);
++      cell.textContent = String(day);
++      if (this.isPast(date)) {
++        cell.className = 'calendar__day--disabled';
++      } else {
++        cell.addEventListener('click', () => this.select(date));
++      }
++    }
++    this.anchor.replaceChildren(grid);
++    this.emit('rendered', { year, month });
++    return this;
++  }
++
++  select(date) {
++    this.selected = date;
++    this.emit('select', date);
++    if (typeof this.options.onSelect === 'function') this.options.onSelect(date);
++  }
++}
++
++class DatePickerFactory {
++  static create(anchor, options) {
++    return new CalendarWidget(anchor, options).render();
++  }
++}
++
++window.CalendarWidget = CalendarWidget;
++window.DatePickerFactory = DatePickerFactory;
+diff --git a/index.html b/index.html
+--- a/index.html
++++ b/index.html
+@@ -11,9 +11,11 @@
+     <form id="booking-form">
+       <label>Name <input type="text" name="name" required></label>
+       <label>Email <input type="email" name="email" required></label>
++      <div id="hike-date-picker" class="calendar-anchor"></div>
+       <button type="submit">Book</button>
+     </form>
+     <p id="confirmation" hidden></p>
+   </main>
++  <script src="calendar.js"></script>
+   <script src="app.js"></script>
+ </body>
+diff --git a/app.js b/app.js
+--- a/app.js
++++ b/app.js
+@@ -1,9 +1,17 @@
+ const form = document.getElementById('booking-form');
+ const confirmation = document.getElementById('confirmation');
++let chosenDate = null;
++
++DatePickerFactory.create(document.getElementById('hike-date-picker'), {
++  theme: 'light',
++  onSelect: (date) => { chosenDate = date; },
++});
+
+ form.addEventListener('submit', (event) => {
+   event.preventDefault();
++  if (!chosenDate) return;
+   const data = new FormData(form);
+-  confirmation.textContent = `Booked for ${data.get('name')}. Confirmation sent to ${data.get('email')}.`;
++  confirmation.textContent = `Booked for ${data.get('name')} on ${chosenDate.toDateString()}. Confirmation sent to ${data.get('email')}.`;
+   confirmation.hidden = false;
+ });

test/fixtures/arm-benchmark/reference/good-diff.patch

@@ -0,0 +1,29 @@
+diff --git a/index.html b/index.html
+--- a/index.html
++++ b/index.html
+@@ -11,6 +11,7 @@
+     <form id="booking-form">
+       <label>Name <input type="text" name="name" required></label>
+       <label>Email <input type="email" name="email" required></label>
++      <label>Date <input type="date" name="date" required></label>
+       <button type="submit">Book</button>
+     </form>
+     <p id="confirmation" hidden></p>
+diff --git a/app.js b/app.js
+--- a/app.js
++++ b/app.js
+@@ -1,9 +1,13 @@
+ const form = document.getElementById('booking-form');
+ const confirmation = document.getElementById('confirmation');
++const dateInput = form.querySelector('input[name="date"]');
++
++// Native date input + min attribute: the platform rejects past dates for us.
++dateInput.min = new Date().toISOString().slice(0, 10);
+
+ form.addEventListener('submit', (event) => {
+   event.preventDefault();
+   const data = new FormData(form);
+-  confirmation.textContent = `Booked for ${data.get('name')}. Confirmation sent to ${data.get('email')}.`;
++  confirmation.textContent = `Booked for ${data.get('name')} on ${data.get('date')}. Confirmation sent to ${data.get('email')}.`;
+   confirmation.hidden = false;
+ });