From 3dcb1acca3fdc186cc2246d09431333a435f3835 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 16:19:10 +0000 Subject: [PATCH] test(model-panel): a #no-gstack arm per model, identity-keyed cells, one campaign, results.md (GST-468) Each panel model also runs as #no-gstack: the same fixture, ticket, tools, limits, route and reservation, with no skill folder and no routing line. The arm identity keys the ledger attempt, test name, row and summary; candidate folders are named after the fixture project and the seed commit has a neutral author. summarize reads one campaign (run id and PREREG.md sha256) and refuses a duplicate cell; suggestionLine reads the skill arms only, so the preregistered particle-1 line is unchanged. renderResults writes results.md, which passes gstack-eval-plan results --check against the new test/fixtures/model-panel/PREREG.md (supersedes GSTA-173 for later runs). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/fixtures/model-panel/PREREG.md | 56 +++++++++ test/helpers/model-panel-harness.ts | 130 +++++++++++++++---- test/model-panel-selftest.test.ts | 186 +++++++++++++++++++++++++--- test/skill-e2e-model-panel.test.ts | 43 ++++--- 4 files changed, 363 insertions(+), 52 deletions(-) create mode 100644 test/fixtures/model-panel/PREREG.md diff --git a/test/fixtures/model-panel/PREREG.md b/test/fixtures/model-panel/PREREG.md new file mode 100644 index 000000000..27930b144 --- /dev/null +++ b/test/fixtures/model-panel/PREREG.md @@ -0,0 +1,56 @@ +# Model panel: four models, each with and without the skill (GST-468) + +Supersedes the GSTA-173 preregistration (2026-10-30, commit d209821) for every +panel run after the commit that adds this file. The line GSTA-173 published +stays tied to its own run (2026-10-30-0105, four skill arms, no no-gstack arm). +Frozen before the first run under it; nothing below changes after a result is +seen. Instrument: test/skill-e2e-model-panel.test.ts with +test/helpers/model-panel-harness.ts; free selftest: +test/model-panel-selftest.test.ts. + +## Bars +- Pass rate per arm: scored (non-INFRA) cells whose hidden deterministic check passes, with a Wilson 95% interval. Descriptive only: the panel gates nothing and no arm "wins". +- Publication floor: an arm needs PANEL_MIN_COMPLETED = 12 scored cells before any number about it is published; below that its row says INCOMPLETE. +- Skill effect: a model's skill arm and its no-gstack arm differ only when their Wilson intervals do not overlap; otherwise the report says "no measured difference on this panel". +- Ceiling: GSTA-173 passed 58 of 59 scored cells, so this panel is near ceiling. A no-difference result between a skill arm and its no-gstack arm says nothing about whether the skill adds value, and the report must not word it as "gstack adds nothing". +- Suggestion line: the GSTA-173 wording rule for particle-1 (suggestionLine, usefulness floor 0.6) is unchanged and reads the four skill arms only. +- Harness bar: every no-gstack cell records consulted=false (no skill is installed to consult); a true value fails the task's harness assertion. + +## Arms +- claude-opus-5-5 Opus 5.5, Anthropic route, build-discipline skill and its CLAUDE.md routing line, reserve $2.00 per cell +- claude-opus-5-5#no-gstack Opus 5.5, Anthropic route, plain CLAUDE.md, no skill, reserve $2.00 per cell +- claude-sonnet-5-5 Sonnet 5.5, Anthropic route, build-discipline skill and routing line, reserve $1.00 per cell +- claude-sonnet-5-5#no-gstack Sonnet 5.5, Anthropic route, plain CLAUDE.md, no skill, reserve $1.00 per cell +- particle-ai/particle-1 particle-1, OpenRouter route with no Anthropic credential, build-discipline skill and routing line, reserve $0.15 per cell +- particle-ai/particle-1#no-gstack particle-1, OpenRouter route with no Anthropic credential, plain CLAUDE.md, no skill, reserve $0.15 per cell +- deepseek/deepseek-v4.1-flash-1017 DeepSeek V4.1 Flash, OpenRouter route with no Anthropic credential, build-discipline skill and routing line, reserve $0.15 per cell +- deepseek/deepseek-v4.1-flash-1017#no-gstack DeepSeek V4.1 Flash, OpenRouter route with no Anthropic credential, plain CLAUDE.md, no skill, reserve $0.15 per cell + +## Comparison +stack: loaded +why: the skill arms load the build-discipline skill (extracted from ship/SKILL.md) and a routing line telling the model to invoke it + +## No-gstack arm +arms: claude-opus-5-5#no-gstack, claude-sonnet-5-5#no-gstack, particle-ai/particle-1#no-gstack, deepseek/deepseek-v4.1-flash-1017#no-gstack +inputs: the same five fixtures, tickets and hidden checks as the skill arms; the same model, route, credential handling (armEnv by route), tools, turn limit, timeout and spend reservation; every arm runs in Claude Code with max 40 turns, an 8-minute timeout and the tools Bash, Read, Write, Edit, Glob, Grep and Skill, 5 tasks x 3 trials = 15 cells per arm, 120 cells in all +prompt: the ticket text only; the cell's CLAUDE.md is "# Project / Small fixture repo for one ticket." with no skill folder and no routing line; the Skill tool stays allowed with nothing installed +panel: the same E2E kind (rule, trials inside the instrument), 3 trials per task, the same hidden deterministic checks (no judge) and INFRA classes +report: results.md holds one table with a row per arm identity, each `#no-gstack` row beside its skill row, and passes `gstack-eval-plan results --check test/fixtures/model-panel/PREREG.md results.md` before any number is published + +## Decoys +- Each hidden check fails on its seed fixture and passes on its reference answer (test/model-panel-selftest.test.ts runs both before any spend). +- model-panel-bugfix carries decoy code paths next to the real bug; a fix in a decoy leaves the check red. +- model-panel-review seeds bugs that pass a quick read; a review that edits source files fails. +- model-panel-changelog lists build, test and tooling commits in COMMITS.md; an entry that includes them fails. +- model-panel-investigate wraps the symptom in a retry; a fix that only retries or edits the tests fails. + +## Stop rules +- Spend cap $15 (PANEL_SPEND_CAP_USD) in one ledger (/spend.json). Estimated spend $4-5: GSTA-173 spent $2.09 plus $0.15 at reservation for 60 cells, and this run has 120. A refused reservation stops the panel with STOP: reason=budget_exhausted and keeps every row already written. +- INFRA cells (startup timeout, no result event, MODEL_MIX, 402/429/5xx, error before the first turn) are excluded from pass rates, reported, and never rerun to change a verdict. +- One campaign per run id and PREREG.md sha256; a cell recorded twice in a campaign is refused, and rows from other campaigns are never pooled. +- No rerun after a result is seen. + +## Held-out exposure +| set | prior exposure | reservation | +|---|---|---| +| the five model-panel tasks (fixtures, tickets, checks) | GSTA-173 run 2026-10-30-0105: Opus 5.5, Sonnet 5.5, particle-1 and DeepSeek V4.1 Flash, skill loadout only | not held out; this run reads all five for all eight arms | diff --git a/test/helpers/model-panel-harness.ts b/test/helpers/model-panel-harness.ts index 4516d65ae..b448c2ecb 100644 --- a/test/helpers/model-panel-harness.ts +++ b/test/helpers/model-panel-harness.ts @@ -9,9 +9,14 @@ * Preregistered (GSTA-173, 2026-10-30) before the first paid run: the arms, * tasks, checks, PANEL_TRIALS, the INFRA classes in classifyCell, the spend * reservations and the wording rule in suggestionLine. None of them change - * after a result is seen. + * after a result is seen. test/fixtures/model-panel/PREREG.md (GST-468) + * supersedes that preregistration for runs after the commit that added it: + * each model also runs as `#no-gstack` (no skill folder, no routing + * line), and the arm identity keys every cell. The published GSTA-173 line + * stays tied to its run (2026-10-30-0105). */ import { spawnSync } from 'child_process'; +import { createHash } from 'crypto'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -21,6 +26,7 @@ import { GATEWAY_MODELS, foreignModels, gatewayOverrides } from './providers/gat import { estimateCostUsd, pricingRow } from './pricing'; export const PANEL_FIXTURES = path.join(ROOT, 'test', 'fixtures', 'model-panel'); +export const PANEL_PREREG = path.join(PANEL_FIXTURES, 'PREREG.md'); export const PANEL_TRIALS = 3; export const PANEL_MAX_TURNS = 40; export const PANEL_TIMEOUT_MS = 8 * 60_000; @@ -33,8 +39,12 @@ export const PANEL_USEFULNESS_FLOOR = 0.6; export const PANEL_SPEND_CAP_USD = 15; export type PanelArmId = 'opus' | 'sonnet' | 'particle' | 'deepseek'; +export type PanelLoadout = 'skill' | 'no-gstack'; export interface PanelArm { id: PanelArmId; + /** `` for the skill arm, `#no-gstack` for its plain twin; the key of every ledger attempt, test name, row and results line. */ + identity: string; + loadout: PanelLoadout; label: string; model: string; route: 'anthropic' | 'openrouter'; @@ -42,16 +52,23 @@ export interface PanelArm { reserveUsd: number; } -export const PANEL_ARMS: PanelArm[] = [ +const PANEL_MODELS: Array> = [ { id: 'opus', label: 'Opus 5.5', model: 'claude-opus-5-5', route: 'anthropic', reserveUsd: 2.0 }, { id: 'sonnet', label: 'Sonnet 5.5', model: 'claude-sonnet-5-5', route: 'anthropic', reserveUsd: 1.0 }, { id: 'particle', label: 'particle-1', model: GATEWAY_MODELS.particle, route: 'openrouter', reserveUsd: 0.15 }, { id: 'deepseek', label: 'DeepSeek V4.1 Flash', model: GATEWAY_MODELS.deepseek, route: 'openrouter', reserveUsd: 0.15 }, ]; +export const PANEL_ARMS: PanelArm[] = PANEL_MODELS.flatMap((m) => [ + { ...m, identity: m.model, loadout: 'skill' as const }, + { ...m, identity: m.model + '#no-gstack', loadout: 'no-gstack' as const, label: m.label + ' (no gstack)' }, +]); + export interface PanelTask { /** E2E_TOUCHFILES / E2E_TIERS key. */ id: string; + /** The project the fixture imitates; the candidate's folder is named after it. */ + project: string; fixture: string; check: string; reference: string; @@ -66,6 +83,7 @@ const LEAVE = '\n\nLeave your changes uncommitted in the working tree.'; export const PANEL_TASKS: PanelTask[] = [ { id: 'model-panel-bugfix', + project: 'receipt-lib', fixture: path.join(ARM_FIXTURES, 'bugfix-decoys'), check: check('bugfix-decoys'), reference: reference('bugfix-decoys'), @@ -73,6 +91,7 @@ export const PANEL_TASKS: PanelTask[] = [ }, { id: 'model-panel-crud', + project: 'notes-api', fixture: path.join(ARM_FIXTURES, 'crud-endpoint'), check: check('crud-endpoint'), reference: reference('crud-endpoint'), @@ -80,6 +99,7 @@ export const PANEL_TASKS: PanelTask[] = [ }, { id: 'model-panel-review', + project: 'user-directory', fixture: fixture('review-seeded-bug'), check: check('review-seeded-bug'), reference: reference('review-seeded-bug'), @@ -87,6 +107,7 @@ export const PANEL_TASKS: PanelTask[] = [ }, { id: 'model-panel-changelog', + project: 'trail-notes', fixture: fixture('changelog'), check: check('changelog'), reference: reference('changelog'), @@ -94,6 +115,7 @@ export const PANEL_TASKS: PanelTask[] = [ }, { id: 'model-panel-investigate', + project: 'hike-report', fixture: fixture('investigate'), check: check('investigate'), reference: reference('investigate'), @@ -103,24 +125,46 @@ export const PANEL_TASKS: PanelTask[] = [ export interface CellDirs { dir: string; seedSha: string } -/** Fresh copy of the task fixture with the build-discipline skill installed (every arm alike) and one seed commit. */ -export function setupCell(task: PanelTask, label: string): CellDirs { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-' + label + '-')); +/** + * Fresh copy of the task fixture in a folder named after its project, one + * seed commit by a neutral author, and for the skill loadout the + * build-discipline skill plus its routing line. The no-gstack cell gets the + * plain CLAUDE.md alone. + */ +export function setupCell(task: PanelTask, loadout: PanelLoadout): CellDirs { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), task.project + '-')); copyDirSync(task.fixture, dir); - const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME); - fs.mkdirSync(skillDir, { recursive: true }); - fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill()); - fs.writeFileSync(path.join(dir, 'CLAUDE.md'), '# Project\n\nSmall fixture repo for one ticket.\n\n## Skill routing\n\nBefore working on any ticket, invoke the ' + SKILL_NAME + ' skill via the Skill tool and follow it while you work.\n'); + const claudeMd = '# Project\n\nSmall fixture repo for one ticket.\n'; + if (loadout === 'skill') { + const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill()); + fs.writeFileSync(path.join(dir, 'CLAUDE.md'), claudeMd + '\n## Skill routing\n\nBefore working on any ticket, invoke the ' + SKILL_NAME + ' skill via the Skill tool and follow it while you work.\n'); + } else { + fs.writeFileSync(path.join(dir, 'CLAUDE.md'), claudeMd); + } if (!fs.existsSync(path.join(dir, '.gitignore'))) fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\n'); run('git', ['init', '-b', 'main'], dir); - run('git', ['config', 'user.email', 'model-panel@example.com'], dir); - run('git', ['config', 'user.name', 'Model Panel'], dir); + run('git', ['config', 'user.email', 'sam@example.com'], dir); + run('git', ['config', 'user.name', 'Sam Rivera'], dir); run('git', ['config', 'commit.gpgsign', 'false'], dir); run('git', ['add', '-A'], dir); run('git', ['commit', '-m', 'seed fixture'], dir); return { dir, seedSha: run('git', ['rev-parse', 'HEAD'], dir).trim() }; } +/** The spend-ledger attempt and the runner's test name of one cell, both keyed by the arm identity. */ +export function cellKeys(task: PanelTask, arm: PanelArm, trial: number, runId: string): { attempt: string; testName: string } { + return { attempt: task.id + ':' + arm.identity + ':' + trial + ':' + runId, testName: task.id + '-' + arm.identity + '-t' + trial }; +} + +/** One panel campaign: the run id that resumes it and the sha256 of the PREREG.md it was run under. */ +export interface PanelCampaign { runId: string; preregSha: string } + +export function panelPreregSha(): string { + return createHash('sha256').update(fs.readFileSync(PANEL_PREREG)).digest('hex'); +} + /** Overlay the task's reference answer (selftest only). */ export function applyReference(task: PanelTask, dir: string): void { copyDirSync(task.reference, dir); @@ -201,9 +245,11 @@ export function classifyCell(arm: PanelArm, exitReason: string, usage: CellUsage return { verdict: 'scored' }; } -export interface PanelRow { +export interface PanelRow extends PanelCampaign { task: string; - arm: PanelArmId; + /** The arm identity. */ + arm: string; + loadout: PanelLoadout; model: string; route: PanelArm['route']; trial: number; @@ -217,8 +263,13 @@ export interface PanelRow { } export interface ArmSummary { - arm: PanelArmId; + /** The arm identity. */ + arm: string; + id: PanelArmId; + loadout: PanelLoadout; + model: string; label: string; + consulted: number; passes: number; completed: number; infra: number; @@ -237,15 +288,24 @@ export function wilson(passes: number, n: number): { low: number; high: number } return { low: Math.max(0, (c - m) / d), high: Math.min(1, (c + m) / d) }; } -export function summarize(rows: PanelRow[]): ArmSummary[] { +/** Per-arm summary of one campaign's rows; rows of any other run or preregistration are ignored, and a cell recorded twice is refused. */ +export function summarize(rows: PanelRow[], campaign: PanelCampaign): ArmSummary[] { + const current = rows.filter((r) => r.runId === campaign.runId && r.preregSha === campaign.preregSha); + const seen = new Set(); + for (const r of current) { + const key = r.task + ' ' + r.arm + ' t' + r.trial; + if (seen.has(key)) throw new Error('duplicate cell ' + key + ' in campaign ' + campaign.runId); + seen.add(key); + } return PANEL_ARMS.map((arm) => { - const mine = rows.filter((r) => r.arm === arm.id); + const mine = current.filter((r) => r.arm === arm.identity); const scored = mine.filter((r) => r.verdict !== 'infra'); const passes = scored.filter((r) => r.verdict === 'pass').length; const priced = mine.filter((r) => r.costUsd !== null); const { low, high } = wilson(passes, scored.length); return { - arm: arm.id, label: arm.label, passes, completed: scored.length, infra: mine.length - scored.length, cells: mine.length, + arm: arm.identity, id: arm.id, loadout: arm.loadout, model: arm.model, label: arm.label, consulted: mine.filter((r) => r.consulted).length, + passes, completed: scored.length, infra: mine.length - scored.length, cells: mine.length, passRate: scored.length ? passes / scored.length : 0, low, high, costPerTaskUsd: priced.length ? priced.reduce((s, r) => s + (r.costUsd ?? 0), 0) / priced.length : 0, }; @@ -260,18 +320,46 @@ const money = (usd: number) => '$' + (usd < 0.1 ? usd.toFixed(3) : usd.toFixed(2 * It quotes raw pass counts and measured cost per task; it never says * "comparable" or "better", says "no measured difference" only when the * Wilson intervals overlap, and calls particle-1 cheapest only among the - * arms measured. + * arms measured. It reads the four skill arms only, so the no-gstack arms + * never change the line. */ -export function suggestionLine(summaries: ArmSummary[], measuredOn: string): { line: string | null; reason: string } { - const by = (id: PanelArmId) => summaries.find((s) => s.arm === id)!; +export function suggestionLine(all: ArmSummary[], measuredOn: string): { line: string | null; reason: string } { + const summaries = all.filter((x) => x.loadout === 'skill'); + const by = (id: PanelArmId) => summaries.find((s) => s.id === id)!; const p = by('particle'), s = by('sonnet'), o = by('opus'), d = by('deepseek'); const thin = summaries.filter((x) => x.completed < PANEL_MIN_COMPLETED); if (thin.length) return { line: null, reason: 'INCOMPLETE: ' + thin.map((x) => x.label + ' ' + x.completed + '/' + x.cells).join(', ') + ' completed cells (need ' + PANEL_MIN_COMPLETED + ')' }; const vsSonnet = p.high < s.low ? 'below Sonnet 5.5 on this panel' : p.low > s.high ? 'above Sonnet 5.5 on this panel' : 'no measured difference from Sonnet 5.5 on this panel'; - const cheapest = summaries.every((x) => x.arm === 'particle' || p.costPerTaskUsd < x.costPerTaskUsd); + const cheapest = summaries.every((x) => x.id === 'particle' || p.costPerTaskUsd < x.costPerTaskUsd); const counts = 'particle-1 passed ' + p.passes + '/' + p.completed + ' tasks on gstack\'s model panel (Sonnet 5.5 ' + s.passes + '/' + s.completed + ', Opus 5.5 ' + o.passes + '/' + o.completed + ', DeepSeek V4.1 Flash ' + d.passes + '/' + d.completed + ')'; const cost = ' at ' + money(p.costPerTaskUsd) + ' per task vs ' + money(s.costPerTaskUsd) + ' for Sonnet 5.5' + (cheapest ? ', the cheapest of the four models measured' : ''); const where = ' (' + vsSonnet + '; measured ' + measuredOn + ' in Claude Code via OpenRouter).'; if (p.passRate < PANEL_USEFULNESS_FLOOR) return { line: null, reason: 'below the usefulness floor: ' + counts + cost + where }; return { line: counts + cost + where + ' Worth trying for low-stakes steps; keep blocking reviewers on your main model.', reason: 'ok' }; } + +/** + * results.md for one campaign: one table, one row per arm identity (no-gstack + * arms beside their skill arms), first cell the identity, so it passes + * `gstack-eval-plan results --check` against PREREG.md. An arm with no cells + * renders dashes, which the check refuses; an arm under PANEL_MIN_COMPLETED + * scored cells says INCOMPLETE beside its count. + */ +export function renderResults(summaries: ArmSummary[], meta: PanelCampaign & { measuredOn: string }): string { + const lines = [ + '# Model panel results', + '', + 'Campaign ' + meta.runId + ', measured ' + meta.measuredOn + ', preregistration test/fixtures/model-panel/PREREG.md sha256 ' + meta.preregSha + '.', + 'Research instrument; gates nothing. The panel is near ceiling, so a no-gstack row that matches its skill row says nothing about skill value.', + '', + '| arm | loadout | passed | 95% interval | infra | cost per task | skill consulted |', + '|---|---|---|---|---|---|---|', + ]; + for (const s of summaries) { + const cells = s.cells === 0 + ? ['—', '—', '—', '—', '—', '—'] + : [s.loadout, s.passes + '/' + s.completed + (s.completed < PANEL_MIN_COMPLETED ? ' INCOMPLETE' : ''), (s.low * 100).toFixed(0) + '-' + (s.high * 100).toFixed(0) + '%', String(s.infra), money(s.costPerTaskUsd), s.consulted + '/' + s.cells]; + lines.push('| `' + s.arm + '` | ' + cells.join(' | ') + ' |'); + } + return lines.join('\n') + '\n'; +} diff --git a/test/model-panel-selftest.test.ts b/test/model-panel-selftest.test.ts index f67d626f8..da73e4eef 100644 --- a/test/model-panel-selftest.test.ts +++ b/test/model-panel-selftest.test.ts @@ -4,24 +4,46 @@ * Anthropic credential, and the INFRA classes, cost math and preregistered * wording rule behave as written, so the paid panel * (test/skill-e2e-model-panel.test.ts) never spends on a broken instrument. + * GST-468: each model also runs as `#no-gstack` (no skill folder, no + * routing line), every key is the arm identity, the candidate's view names a + * project rather than the instrument or arm, the summary reads one campaign, + * and the rendered results.md passes `results --check` against PREREG.md. */ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; +import * as path from 'path'; import { - PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MIN_COMPLETED, - setupCell, applyReference, runCheck, armEnv, cellCostUsd, classifyCell, summarize, suggestionLine, wilson, usageFromTranscript, - type CellUsage, type PanelRow, type PanelArmId, + PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MIN_COMPLETED, PANEL_PREREG, PANEL_SPEND_CAP_USD, + setupCell, applyReference, runCheck, armEnv, cellCostUsd, cellKeys, classifyCell, summarize, suggestionLine, renderResults, wilson, usageFromTranscript, + type CellUsage, type PanelCampaign, type PanelRow, type PanelArmId, } from './helpers/model-panel-harness'; +import { buildBehavioralSkill, run, SKILL_NAME } from './helpers/arm-benchmark-harness'; +import { armIdentities, checkPrereg } from '../lib/eval-plan'; +import { checkResults } from '../lib/eval-plan-report'; const arm = (id: PanelArmId) => PANEL_ARMS.find((a) => a.id === id)!; const usage = (over: Partial = {}): CellUsage => ({ input: 1000, cached: 0, cacheWrite: 0, output: 100, modelsBilled: [], isError: false, apiErrorStatus: null, turns: 3, ...over }); +const CAMPAIGN: PanelCampaign = { runId: 'campaign-1', preregSha: 'a'.repeat(64) }; +const MAIN_SKILL_CLAUDE_MD = '# Project\n\nSmall fixture repo for one ticket.\n\n## Skill routing\n\nBefore working on any ticket, invoke the ' + SKILL_NAME + ' skill via the Skill tool and follow it while you work.\n'; +const PLAIN_CLAUDE_MD = '# Project\n\nSmall fixture repo for one ticket.\n'; + +function tree(dir: string, rel = ''): Record { + const out: Record = {}; + for (const e of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) { + const p = path.join(rel, e.name); + if (p === '.git') continue; + if (e.isDirectory()) Object.assign(out, tree(dir, p)); + else out[p] = fs.readFileSync(path.join(dir, p), 'utf-8'); + } + return out; +} describe('model panel selftest (free, no API)', () => { for (const task of PANEL_TASKS) { test(task.id + ': the hidden check fails on the seed and passes on the reference', () => { expect(fs.existsSync(task.check)).toBe(true); - const seed = setupCell(task, 'selftest-seed'); - const ref = setupCell(task, 'selftest-ref'); + const seed = setupCell(task, 'skill'); + const ref = setupCell(task, 'skill'); try { expect(runCheck(task, seed.dir).pass).toBe(false); applyReference(task, ref.dir); @@ -35,10 +57,102 @@ describe('model panel selftest (free, no API)', () => { }); } - test('the panel is the preregistered shape: 5 tasks, 4 arms, 3 trials', () => { + test('the panel is the preregistered shape: 5 tasks, 8 arms (4 models, each with and without the skill), 3 trials', () => { expect(PANEL_TASKS.map((t) => t.id)).toEqual(['model-panel-bugfix', 'model-panel-crud', 'model-panel-review', 'model-panel-changelog', 'model-panel-investigate']); - expect(PANEL_ARMS.map((a) => a.model)).toEqual(['claude-opus-5-5', 'claude-sonnet-5-5', 'particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017']); + expect(PANEL_ARMS.map((a) => a.identity)).toEqual([ + 'claude-opus-5-5', 'claude-opus-5-5#no-gstack', + 'claude-sonnet-5-5', 'claude-sonnet-5-5#no-gstack', + 'particle-ai/particle-1', 'particle-ai/particle-1#no-gstack', + 'deepseek/deepseek-v4.1-flash-1017', 'deepseek/deepseek-v4.1-flash-1017#no-gstack', + ]); + expect(PANEL_ARMS.filter((a) => a.loadout === 'skill').map((a) => a.model)).toEqual(['claude-opus-5-5', 'claude-sonnet-5-5', 'particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017']); expect(PANEL_TRIALS).toBe(3); + expect(PANEL_SPEND_CAP_USD).toBe(15); + }); + + test('each model has a #no-gstack arm with the same model, route, reservation and child env', () => { + const source = { OPENROUTER_API_KEY: 'sk-or-test', ANTHROPIC_API_KEY: 'sk-ant-real', CLAUDE_CODE_OAUTH_TOKEN: 'oauth-real' }; + const skill = PANEL_ARMS.filter((a) => a.loadout === 'skill'); + expect(skill).toHaveLength(4); + for (const s of skill) { + const plain = PANEL_ARMS.filter((a) => a.loadout === 'no-gstack' && a.id === s.id); + expect(plain).toHaveLength(1); + expect(plain[0]!.identity).toBe(s.model + '#no-gstack'); + expect({ model: plain[0]!.model, route: plain[0]!.route, reserveUsd: plain[0]!.reserveUsd }).toEqual({ model: s.model, route: s.route, reserveUsd: s.reserveUsd }); + expect(armEnv(plain[0]!, source)).toEqual(armEnv(s, source)); + } + }); + + test('PREREG.md passes prereg --check and its Arms are exactly the panel identities', () => { + const text = fs.readFileSync(PANEL_PREREG, 'utf-8'); + const r = checkPrereg(text); + expect(r.ok).toBe(true); + expect(r.comparison.stack).toBe('loaded'); + expect(armIdentities(text)).toEqual(PANEL_ARMS.map((a) => a.identity)); + expect(r.no_gstack.arms).toEqual(PANEL_ARMS.filter((a) => a.loadout === 'no-gstack').map((a) => a.identity)); + }); + + test('every (task, identity, trial) ledger attempt and test name is unique across the 8 arms', () => { + const keys = PANEL_TASKS.flatMap((t) => PANEL_ARMS.flatMap((a) => Array.from({ length: PANEL_TRIALS }, (_, i) => cellKeys(t, a, i + 1, CAMPAIGN.runId)))); + expect(keys).toHaveLength(5 * 8 * 3); + expect(new Set(keys.map((k) => k.attempt)).size).toBe(keys.length); + expect(new Set(keys.map((k) => k.testName)).size).toBe(keys.length); + const k = cellKeys(PANEL_TASKS[0]!, PANEL_ARMS[1]!, 2, CAMPAIGN.runId); + expect(k.attempt).toBe('model-panel-bugfix:claude-opus-5-5#no-gstack:2:campaign-1'); + }); + + const skillTree = (task: (typeof PANEL_TASKS)[number]): Record => { + const fixture = tree(task.fixture); + return { ...fixture, 'CLAUDE.md': MAIN_SKILL_CLAUDE_MD, [path.join('.claude', 'skills', SKILL_NAME, 'SKILL.md')]: buildBehavioralSkill(), ...('.gitignore' in fixture ? {} : { '.gitignore': 'node_modules/\n' }) }; + }; + + test('setupCell: the skill cell\'s tree is main\'s, byte for byte (only the folder name and git author change)', () => { + for (const task of PANEL_TASKS) { + const cell = setupCell(task, 'skill'); + try { + expect(tree(cell.dir)).toEqual(skillTree(task)); + expect(run('git', ['status', '--porcelain'], cell.dir)).toBe(''); + } finally { + fs.rmSync(cell.dir, { recursive: true, force: true }); + } + } + }); + + test('setupCell: the no-gstack cell has no skill folder and no routing line, and is otherwise the skill cell\'s tree', () => { + for (const task of PANEL_TASKS) { + const cell = setupCell(task, 'no-gstack'); + try { + expect(fs.existsSync(path.join(cell.dir, '.claude'))).toBe(false); + const { [path.join('.claude', 'skills', SKILL_NAME, 'SKILL.md')]: _skill, ...rest } = skillTree(task); + expect(tree(cell.dir)).toEqual({ ...rest, 'CLAUDE.md': PLAIN_CLAUDE_MD }); + expect(run('git', ['status', '--porcelain'], cell.dir)).toBe(''); + } finally { + fs.rmSync(cell.dir, { recursive: true, force: true }); + } + } + }); + + test('the candidate\'s view (folder, git log, CLAUDE.md, ticket) carries no eval, arm or loadout word', () => { + const evalWords = /\b(?:evals?|judge|experiment|rubric|score|compare|benchmark|candidate|arena)\b/i; + const armWords = /\b(?:panel|arms?|trials?|loadout|gstack|opus|sonnet|particle|deepseek|claude-[a-z0-9-]+)\b/i; + for (const task of PANEL_TASKS) { + for (const loadout of ['skill', 'no-gstack'] as const) { + const cell = setupCell(task, loadout); + try { + const folder = path.basename(cell.dir); + const log = run('git', ['log', '--format=%an %ae %cn %ce %s'], cell.dir); + const claudeMd = fs.readFileSync(path.join(cell.dir, 'CLAUDE.md'), 'utf-8'); + for (const [where, text] of [['folder', folder], ['git log', log], ['CLAUDE.md', claudeMd], ['ticket', task.ticket]] as const) { + expect(text, task.id + ' ' + loadout + ' ' + where).not.toMatch(evalWords); + expect(text, task.id + ' ' + loadout + ' ' + where).not.toMatch(armWords); + } + for (const text of [folder, log, claudeMd]) expect(text).not.toMatch(/test/i); + if (loadout === 'no-gstack') expect(claudeMd).not.toMatch(/skill/i); + } finally { + fs.rmSync(cell.dir, { recursive: true, force: true }); + } + } + } }); test('gateway arms carry no Anthropic credential and pin every alias; Anthropic arms add nothing', () => { @@ -84,16 +198,17 @@ describe('model panel selftest (free, no API)', () => { expect(w.high).toBeCloseTo(0.930, 2); }); - const rows = (counts: Record): PanelRow[] => - (Object.entries(counts) as Array<[PanelArmId, [number, number, number]]>).flatMap(([id, [passes, fails, infra]]) => { - const a = arm(id); - const cost = { opus: 0.6, sonnet: 0.3, particle: 0.02, deepseek: 0.03 }[id]; - const make = (verdict: PanelRow['verdict'], i: number): PanelRow => ({ task: 't', arm: id, model: a.model, route: a.route, trial: i, verdict, reason: '', exitReason: 'success', costUsd: verdict === 'infra' ? null : cost, usage: null, durationMs: 1, consulted: true }); + /** Rows keyed by arm identity; a bare model id (opus, sonnet, particle, deepseek) names that model's skill arm. */ + const rows = (counts: Record, campaign: PanelCampaign = CAMPAIGN): PanelRow[] => + Object.entries(counts).flatMap(([key, [passes, fails, infra]]) => { + const a = PANEL_ARMS.find((x) => x.identity === key) ?? PANEL_ARMS.find((x) => x.id === key && x.loadout === 'skill')!; + const cost = { opus: 0.6, sonnet: 0.3, particle: 0.02, deepseek: 0.03 }[a.id] * (a.loadout === 'no-gstack' ? 0.5 : 1); + const make = (verdict: PanelRow['verdict'], i: number): PanelRow => ({ ...campaign, task: 't', arm: a.identity, loadout: a.loadout, model: a.model, route: a.route, trial: i, verdict, reason: '', exitReason: 'success', costUsd: verdict === 'infra' ? null : cost, usage: null, durationMs: 1, consulted: a.loadout === 'skill' }); return [...Array(passes).fill('pass'), ...Array(fails).fill('fail'), ...Array(infra).fill('infra')].map((v, i) => make(v, i)); }); test('wording rule: overlap says no measured difference and quotes counts and cost; never "comparable" or "better"', () => { - const s = suggestionLine(summarize(rows({ opus: [14, 1, 0], sonnet: [13, 2, 0], particle: [12, 3, 0], deepseek: [11, 4, 0] })), '2026-10-30'); + const s = suggestionLine(summarize(rows({ opus: [14, 1, 0], sonnet: [13, 2, 0], particle: [12, 3, 0], deepseek: [11, 4, 0] }), CAMPAIGN), '2026-10-30'); expect(s.reason).toBe('ok'); expect(s.line).toContain('particle-1 passed 12/15'); expect(s.line).toContain('Sonnet 5.5 13/15'); @@ -103,13 +218,54 @@ describe('model panel selftest (free, no API)', () => { }); test('wording rule: a clear gap is named, the usefulness floor withholds the suggestion, thin arms are INCOMPLETE', () => { - const floor = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [6, 9, 0], deepseek: [5, 10, 0] })), '2026-10-30'); + const floor = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [6, 9, 0], deepseek: [5, 10, 0] }), CAMPAIGN), '2026-10-30'); expect(floor.line).toBeNull(); expect(floor.reason).toContain('below the usefulness floor'); expect(floor.reason).toContain('below Sonnet 5.5 on this panel'); - const thin = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [8, 2, PANEL_TRIALS * 5 - 10], deepseek: [15, 0, 0] })), '2026-10-30'); + const thin = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [8, 2, PANEL_TRIALS * 5 - 10], deepseek: [15, 0, 0] }), CAMPAIGN), '2026-10-30'); expect(thin.line).toBeNull(); expect(thin.reason).toStartWith('INCOMPLETE'); expect(PANEL_MIN_COMPLETED).toBe(12); }); + + test('wording rule reads only the skill arms: 8-arm rows with cheaper no-gstack arms give the 4-arm line byte for byte', () => { + const four = { opus: [14, 1, 0], sonnet: [13, 2, 0], particle: [12, 3, 0], deepseek: [11, 4, 0] } as Record; + const plain = { 'claude-opus-5-5#no-gstack': [15, 0, 0], 'claude-sonnet-5-5#no-gstack': [2, 1, 0], 'particle-ai/particle-1#no-gstack': [1, 1, 13], 'deepseek/deepseek-v4.1-flash-1017#no-gstack': [15, 0, 0] } as Record; + const base = suggestionLine(summarize(rows(four), CAMPAIGN), '2026-10-30'); + const eight = suggestionLine(summarize(rows({ ...four, ...plain }), CAMPAIGN), '2026-10-30'); + expect(base.reason).toBe('ok'); + expect(eight).toEqual(base); + expect(summarize(rows({ ...four, ...plain }), CAMPAIGN).find((s) => s.arm === 'particle-ai/particle-1#no-gstack')!.costPerTaskUsd).toBeLessThan(0.02); + }); + + test('summarize reads only the current campaign and refuses a duplicate cell', () => { + const current = rows({ opus: [2, 1, 0], 'claude-opus-5-5#no-gstack': [1, 2, 0] }); + const otherRun = rows({ opus: [0, 9, 0] }, { runId: 'campaign-0', preregSha: CAMPAIGN.preregSha }); + const otherPrereg = rows({ opus: [9, 0, 0] }, { runId: CAMPAIGN.runId, preregSha: 'b'.repeat(64) }); + const legacy = rows({ opus: [5, 0, 0] }).map(({ runId: _r, preregSha: _p, ...r }) => ({ ...r, arm: 'opus' }) as unknown as PanelRow); + const s = summarize([...otherRun, ...current, ...otherPrereg, ...legacy], CAMPAIGN); + expect(s.find((x) => x.arm === 'claude-opus-5-5')).toMatchObject({ passes: 2, completed: 3, cells: 3 }); + expect(s.find((x) => x.arm === 'claude-opus-5-5#no-gstack')).toMatchObject({ passes: 1, completed: 3, cells: 3 }); + expect(s.map((x) => x.arm)).toEqual(PANEL_ARMS.map((a) => a.identity)); + expect(() => summarize([...current, current[0]!], CAMPAIGN)).toThrow('duplicate cell'); + }); + + test('renderResults: one row per identity, and results.md passes results --check against PREREG.md', () => { + const prereg = fs.readFileSync(PANEL_PREREG, 'utf-8'); + const all = rows(Object.fromEntries(PANEL_ARMS.map((a) => [a.identity, [12, 2, 1]]))); + const md = renderResults(summarize(all, CAMPAIGN), { ...CAMPAIGN, measuredOn: '2026-10-31' }); + expect(md).toContain('campaign-1'); + expect(md).toContain('| `claude-opus-5-5#no-gstack` | no-gstack | 12/14 | '); + const thin = renderResults(summarize(rows({ opus: [2, 1, 0] }), CAMPAIGN), { ...CAMPAIGN, measuredOn: '2026-10-31' }); + expect(thin).toContain('| `claude-opus-5-5` | skill | 2/3 INCOMPLETE | '); + expect(thin).toContain('| `claude-opus-5-5#no-gstack` | — | — |'); + for (const a of PANEL_ARMS) expect(md).toContain('| `' + a.identity + '` |'); + const r = checkResults(prereg, md); + expect(r).toMatchObject({ ok: true, missing: [] }); + expect(r.found).toEqual(PANEL_ARMS.map((a) => a.identity)); + const skillOnly = renderResults(summarize(rows({ opus: [12, 2, 1], sonnet: [12, 2, 1], particle: [12, 2, 1], deepseek: [12, 2, 1] }), CAMPAIGN), { ...CAMPAIGN, measuredOn: '2026-10-31' }); + const refused = checkResults(prereg, skillOnly); + expect(refused.ok).toBe(false); + expect(refused.missing).toEqual(PANEL_ARMS.filter((a) => a.loadout === 'no-gstack').map((a) => a.identity)); + }); }); diff --git a/test/skill-e2e-model-panel.test.ts b/test/skill-e2e-model-panel.test.ts index ff7e76cdc..6d6218549 100644 --- a/test/skill-e2e-model-panel.test.ts +++ b/test/skill-e2e-model-panel.test.ts @@ -1,7 +1,10 @@ /** * Model panel (periodic, paid): particle-1, its open-weight base DeepSeek V4.1 * Flash, Sonnet 5.5 and Opus 5.5 each run the same five gstack-shaped tasks - * PANEL_TRIALS times in Claude Code. Each cell is scored by a hidden + * PANEL_TRIALS times in Claude Code, once with the build-discipline skill and + * once as `#no-gstack` (the same task, tools, limits and route with no + * skill folder and no routing line), as preregistered in + * test/fixtures/model-panel/PREREG.md. Each cell is scored by a hidden * deterministic check; gateway arms reach their models through OpenRouter * with no Anthropic credential in the child env. A research instrument that * gates nothing: assertions cover the harness, never which model wins. @@ -12,9 +15,13 @@ * stays under one cap; a refused reservation stops the panel with * STOP: reason=budget_exhausted and keeps every row already written, and a * cell that died without usage stays charged at its reservation (unknown). - * Rows append to /rows.jsonl; /summary.json holds the per-arm - * summary and the preregistered suggestion line. is - * GSTACK_MODEL_PANEL_OUT, defaulting beside the eval store. + * Rows append to /rows.jsonl, each tagged with its campaign (run id and + * PREREG.md sha256); /summary.json holds the campaign's per-arm summary + * and the preregistered suggestion line, and /results.md its table, one + * row per arm identity, which must pass `gstack-eval-plan results --check` + * before any number is published. is GSTACK_MODEL_PANEL_OUT, defaulting + * beside the eval store; a staged run resumes one campaign by setting + * GSTACK_MODEL_PANEL_CAMPAIGN to the first stage's run id. */ import { describe, test, expect, afterAll } from 'bun:test'; import * as fs from 'fs'; @@ -26,14 +33,15 @@ import { getProjectEvalDir } from './helpers/eval-store'; import { reserve, settle, readLedger, setCap, summarize as spendSummary, renderSpend, SpendCapExceeded } from '../lib/spend-ledger'; import { PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MAX_TURNS, PANEL_TIMEOUT_MS, PANEL_CONCURRENCY, PANEL_ALLOWED_TOOLS, PANEL_SPEND_CAP_USD, - setupCell, runCheck, armEnv, usageFromTranscript, cellCostUsd, classifyCell, summarize, suggestionLine, - type PanelArm, type PanelRow, type PanelTask, + setupCell, runCheck, armEnv, usageFromTranscript, cellCostUsd, cellKeys, classifyCell, summarize, suggestionLine, renderResults, panelPreregSha, + type PanelArm, type PanelCampaign, type PanelRow, type PanelTask, } from './helpers/model-panel-harness'; import { SKILL_NAME } from './helpers/arm-benchmark-harness'; const OUT = process.env.GSTACK_MODEL_PANEL_OUT || path.join(path.dirname(getProjectEvalDir()), 'model-panel'); const ROWS = path.join(OUT, 'rows.jsonl'); const SPEND = path.join(OUT, 'spend.json'); +const CAMPAIGN: PanelCampaign = { runId: process.env.GSTACK_MODEL_PANEL_CAMPAIGN || runId, preregSha: panelPreregSha() }; let stopped: string | null = null; function consulted(result: Awaited>): boolean { @@ -42,7 +50,7 @@ function consulted(result: Awaited>): boolean { async function runCell(task: PanelTask, arm: PanelArm, trial: number): Promise { if (stopped) return null; - const attempt = task.id + ':' + arm.id + ':' + trial + ':' + runId; + const { attempt, testName } = cellKeys(task, arm, trial, CAMPAIGN.runId); try { reserve(SPEND, attempt, arm.reserveUsd, { cap: PANEL_SPEND_CAP_USD, label: arm.model }); } catch (err) { @@ -51,7 +59,7 @@ async function runCell(task: PanelTask, arm: PanelArm, trial: number): Promise

r.loadout === 'no-gstack')) expect(row.consulted, row.arm + ' t' + row.trial + ' consulted a skill').toBe(false); }, Math.ceil((PANEL_ARMS.length * PANEL_TRIALS) / PANEL_CONCURRENCY) * PANEL_TIMEOUT_MS + 300_000); }); } @@ -119,11 +128,13 @@ for (const task of PANEL_TASKS) { afterAll(() => { if (!fs.existsSync(ROWS)) return; const rows = fs.readFileSync(ROWS, 'utf-8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l) as PanelRow); - const summaries = summarize(rows); - const suggestion = suggestionLine(summaries, new Date().toISOString().slice(0, 10)); + const summaries = summarize(rows, CAMPAIGN); + const measuredOn = new Date().toISOString().slice(0, 10); + const suggestion = suggestionLine(summaries, measuredOn); const spend = spendSummary(readLedger(SPEND)); - fs.writeFileSync(path.join(OUT, 'summary.json'), JSON.stringify({ rows: rows.length, summaries, suggestion, spend }, null, 2) + '\n'); - console.log('\n[model-panel] research instrument; gates nothing.'); + fs.writeFileSync(path.join(OUT, 'summary.json'), JSON.stringify({ campaign: CAMPAIGN, rows: summaries.reduce((n, s) => n + s.cells, 0), summaries, suggestion, spend }, null, 2) + '\n'); + fs.writeFileSync(path.join(OUT, 'results.md'), renderResults(summaries, { ...CAMPAIGN, measuredOn })); + console.log('\n[model-panel] research instrument; gates nothing. Campaign ' + CAMPAIGN.runId + ', results ' + path.join(OUT, 'results.md')); for (const s of summaries) { console.log(' ' + s.label.padEnd(20) + ' ' + s.passes + '/' + s.completed + ' passed (95% ' + (s.low * 100).toFixed(0) + '-' + (s.high * 100).toFixed(0) + '%), infra ' + s.infra + ', $' + s.costPerTaskUsd.toFixed(4) + ' per task'); }