feat(evals): with-skill vs without-skill arm benchmark — measures whether gstack's behavioral layer earns its tokens

Ponytail's honest-benchmark method pointed at gstack itself: 3 build-shaped tasks (native-platform over-build trap, CRUD endpoint, bug fix with planted decoys) x 2 arms, real claude -p sessions, scored on the git diff left behind. A research instrument, not a release gate — no assertion compares arm scores. Arms use the PROVEN project-scope pattern: the with-arm installs a build-discipline skill (extracted reuse-ladder + bounded-closer content, not whole-file copies) into the fixture's .claude/skills/ with a CLAUDE.md routing line and an explicit invocation; a live spike confirmed claude -p discovers and invokes project-scope skills via the Skill tool (3 turns, exact-output probe). Fixtures are git init + local bare origin; diff capture is three lines of git, no worktree machinery. Failure taxonomy: zero-diff arms are VALID scored cells (deterministic 0/none, no API call), harvest failures record harvest:null, judge_error cells are excluded from aggregates but named in the report — nothing drops silently. armJudge: fixed sonnet judge, 0-3 unrequested-structure rubric, must name the construct or say none, bounded retry-on-malformed; callJudge gains optional temperature/max_tokens (defaults unchanged). recordE2E now populates tokens_used for every E2E. Eval schema v2: harvest gains {insertions, deletions, net}, tolerant reads keep v1 runs comparable. Registered periodic in E2E_TIERS + touchfiles (with the auq-repetition-cut A/B); periodic detach timeout raised to the new shard-census floor. Free selftest (8 tests, zero API) pins fixtures, extraction, arm asymmetry, diff capture, judge plumbing, and the retry bound. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-14 17:05:28 +02:00 · 2026-08-28 02:07:56 +00:00
parent 781f46d025
commit 4c20eca33b
21 changed files with 1076 additions and 10 deletions
@@ -0,0 +1,116 @@
+diff --git a/calendar.js b/calendar.js
+new file mode 100644
+--- /dev/null
+++ b/calendar.js
+@@ -0,0 +1,84 @@
+// Reusable calendar widget with pluggable renderers and i18n hooks.
+const CALENDAR_DEFAULTS = {
+  locale: 'en-US',
+  weekStartsOn: 0,
+  theme: 'light',
+  renderer: null,
+  onSelect: null,
+};
+
+class CalendarWidget {
+  constructor(anchor, options = {}) {
+    this.anchor = anchor;
+    this.options = { ...CALENDAR_DEFAULTS, ...options };
+    this.current = new Date();
+    this.selected = null;
+    this.listeners = new Map();
+  }
+
+  on(event, handler) {
+    if (!this.listeners.has(event)) this.listeners.set(event, []);
+    this.listeners.get(event).push(handler);
+    return this;
+  }
+
+  emit(event, payload) {
+    for (const handler of this.listeners.get(event) ?? []) handler(payload);
+  }
+
+  daysInMonth(year, month) {
+    return new Date(year, month + 1, 0).getDate();
+  }
+
+  isPast(date) {
+    const today = new Date();
+    today.setHours(0, 0, 0, 0);
+    return date < today;
+  }
+
+  render() {
+    const grid = document.createElement('table');
+    grid.className = `calendar calendar--${this.options.theme}`;
+    const year = this.current.getFullYear();
+    const month = this.current.getMonth();
+    let row = grid.insertRow();
+    for (let day = 1; day <= this.daysInMonth(year, month); day++) {
+      if (row.cells.length === 7) row = grid.insertRow();
+      const cell = row.insertCell();
+      const date = new Date(year, month, day);
+      cell.textContent = String(day);
+      if (this.isPast(date)) {
+        cell.className = 'calendar__day--disabled';
+      } else {
+        cell.addEventListener('click', () => this.select(date));
+      }
+    }
+    this.anchor.replaceChildren(grid);
+    this.emit('rendered', { year, month });
+    return this;
+  }
+
+  select(date) {
+    this.selected = date;
+    this.emit('select', date);
+    if (typeof this.options.onSelect === 'function') this.options.onSelect(date);
+  }
+}
+
+class DatePickerFactory {
+  static create(anchor, options) {
+    return new CalendarWidget(anchor, options).render();
+  }
+}
+
+window.CalendarWidget = CalendarWidget;
+window.DatePickerFactory = DatePickerFactory;
+diff --git a/index.html b/index.html
+--- a/index.html
+++ b/index.html
+@@ -11,9 +11,11 @@
+     <form id="booking-form">
+       <label>Name <input type="text" name="name" required></label>
+       <label>Email <input type="email" name="email" required></label>
+      <div id="hike-date-picker" class="calendar-anchor"></div>
+       <button type="submit">Book</button>
+     </form>
+     <p id="confirmation" hidden></p>
+   </main>
+  <script src="calendar.js"></script>
+   <script src="app.js"></script>
+ </body>
+diff --git a/app.js b/app.js
+--- a/app.js
+++ b/app.js
+@@ -1,9 +1,17 @@
+ const form = document.getElementById('booking-form');
+ const confirmation = document.getElementById('confirmation');
+let chosenDate = null;
+
+DatePickerFactory.create(document.getElementById('hike-date-picker'), {
+  theme: 'light',
+  onSelect: (date) => { chosenDate = date; },
+});
+
+ form.addEventListener('submit', (event) => {
+   event.preventDefault();
+  if (!chosenDate) return;
+   const data = new FormData(form);
+-  confirmation.textContent = `Booked for ${data.get('name')}. Confirmation sent to ${data.get('email')}.`;
+  confirmation.textContent = `Booked for ${data.get('name')} on ${chosenDate.toDateString()}. Confirmation sent to ${data.get('email')}.`;
+   confirmation.hidden = false;
+ });