From 74d5588748a05bfa32848af0517b5e7f38025071 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 16:09:27 +0000 Subject: [PATCH] feat(eval-plan): results --check refuses a results table without its no-gstack rows Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-doctor-components.sh | 1 + bin/gstack-eval-plan | 22 +++++- docs/TESTING_INTERNALS.md | 7 +- docs/troubleshooting.md | 15 ++++ eval-plan/SKILL.md | 14 ++++ eval-plan/SKILL.md.tmpl | 14 ++++ lib/eval-plan-report.ts | 46 ++++++++++++ lib/result-codes-eval.ts | 5 ++ test/eval-plan-results.test.ts | 96 ++++++++++++++++++++++++++ test/fixtures/module-size-ratchet.json | 1 + 10 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 lib/eval-plan-report.ts create mode 100644 test/eval-plan-results.test.ts diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 36caac274..d99e9c186 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -347,6 +347,7 @@ gstack_result_anchor() { PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;; PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;; + EVAL_REPORT_NO_GSTACK_ROW) printf '%s' 'eval-report-no-gstack-row' ;; EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;; *) return 1 ;; esac diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 1dc081a76..020494a43 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -10,6 +10,7 @@ * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] + * gstack-eval-plan results --check [--json] * gstack-eval-plan price-gate --gate-file | --gate "" [--plan-file ] [--base ] [--cap-usd N] * [--smoke-cap-usd N] [--allow-unpriced ""] [--out ] [--json] * @@ -26,12 +27,13 @@ * price-gate lint every arm's model against the base branch's * .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete * prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused. - * Logic: lib/eval-plan.ts. + * Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; +import { checkResults } from '../lib/eval-plan-report'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; @@ -41,7 +43,8 @@ import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) prereg --check [--allow-repo-commands] [--json] - the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure; + the required sections exist and are filled: Bars, Arms, Comparison, Decoys, Stop rules, + Held-out exposure, and No-gstack arm when the Comparison says stack: loaded; an optional Diagnostics section's summarizer runs on its synthetic fixture only with --allow-repo-commands (else it is listed and exit 3) and every count must match prereg --init [--title ] write the template @@ -53,6 +56,9 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask + results --check [--json] + one table in the results file has a filled row for every Arms identity, each no-gstack arm + beside its variants; reads text only (docs/TESTING_INTERNALS.md#no-gstack-arm) price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a @@ -129,6 +135,17 @@ function doPrereg(rest: string[]): number { return EXIT.ok; } +function doResults(rest: string[]): number { + const prereg = flag(rest, '--check'); + const results = rest.filter(a => !a.startsWith('--') && a !== prereg)[0]; + if (!prereg || !results || !fs.existsSync(prereg) || !fs.existsSync(results)) return usage('results needs --check '); + const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8')); + if (rest.includes('--json')) process.stdout.write(JSON.stringify({ prereg, results, ...r }) + '\n'); + else process.stdout.write(`RESULTS: ${r.ok ? 'ok' : 'incomplete'} no_gstack_rows=${r.no_gstack.filter(id => r.found.includes(id)).length}/${r.no_gstack.length} rows=${r.found.length}/${r.required.length} file=${results}\n`); + if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan results', 'EVAL_REPORT_NO_GSTACK_ROW', r.problems.join('; ')) + '\n'); return EXIT.fail; } + return EXIT.ok; +} + function doArms(rest: string[]): number { const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! }; const scope = scopeOf(rest); @@ -242,6 +259,7 @@ function main(argv: string[]): number { if (cmd === 'dry-run') return doDryRun(rest); if (cmd === 'pilot') return doPilot(rest); if (cmd === 'price-gate') return doPriceGate(rest); + if (cmd === 'results') return doResults(rest); return usage(`unknown command ${cmd}`); } catch (e: any) { if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; } diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 2ed948c98..bb2e1f2dc 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1676,9 +1676,10 @@ own attempts. The arm gets: **Reporting.** The no-gstack result is one row beside the variants in every results table and every verdict, including when it wins. A comparison that -drops the row has not reported its result. The prereg check cannot see the -report, so the `report:` line states where the row goes and the reviewer of -the result holds the author to it. +drops the row has not reported its result. `gstack-eval-plan results --check + ` enforces it: one table in the results file must hold a +filled row for every Arms identity (`EVAL_REPORT_NO_GSTACK_ROW` otherwise), and +no number is published before it passes. **Precedent and its gaps.** `test/helpers/arm-benchmark-harness.ts` already has a `without-skill` arm beside `with-skill`, and it is the closest existing diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5f1fb6c28..f0a7ce5fe 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3672,6 +3672,21 @@ goes beside the variants. When no arm loads a skill stack, write `stack: none` with a `why:` line instead. The rule is in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm). + +### `EVAL_REPORT_NO_GSTACK_ROW` + +**Meaning.** `gstack-eval-plan results --check ` found no +single markdown table in the results file with a filled row for every arm +identity the preregistration's Arms section lists. Usually the +`#no-gstack` row is missing, sits in a separate table from its +variants, or holds only placeholders (`-`, `n/a`, `tbd`). It also fires when +the preregistration itself fails `prereg --check`. + +**Fix.** Put one row per arm identity, each no-gstack arm beside its variants, +in one table, and rerun the check. Publish no number before it prints +`RESULTS: ok`. A preregistration with `stack: none` needs no row. The rule is +in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm). + ### `EVAL_ARM_DUPLICATE` diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 5ec40e511..66d7d8920 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -307,6 +307,20 @@ Under `SESSION_KIND: unattended` write the question as an `approval` gate item t The full run, when approved, keeps settling every attempt into the same ledger and reprints the spend table at each gate; `budget_exhausted` stops it. +## Step 5: Results beside the no-gstack arm + +Write the run's results as one markdown table in a results file, one row per arm identity +from the Arms section, each `#no-gstack` row beside its variants. Before any number +leaves the run (a report, a READY block, a PR body), check it: + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan results --check +``` + +`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm +whose filled row is missing from the table that holds the others; add it and rerun. A +preregistration with `stack: none` needs no row. + ## Tier 3, after P7 (not in this skill yet) Each of these waits for the adoption number in the multi-agent wave plan (P7); one line each so the shape is known: diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 019499eca..8a2fc4d75 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -222,6 +222,20 @@ Under `SESSION_KIND: unattended` write the question as an `approval` gate item t The full run, when approved, keeps settling every attempt into the same ledger and reprints the spend table at each gate; `budget_exhausted` stops it. +## Step 5: Results beside the no-gstack arm + +Write the run's results as one markdown table in a results file, one row per arm identity +from the Arms section, each `#no-gstack` row beside its variants. Before any number +leaves the run (a report, a READY block, a PR body), check it: + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan results --check +``` + +`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm +whose filled row is missing from the table that holds the others; add it and rerun. A +preregistration with `stack: none` needs no row. + ## Tier 3, after P7 (not in this skill yet) Each of these waits for the adoption number in the multi-agent wave plan (P7); one line each so the shape is known: diff --git a/lib/eval-plan-report.ts b/lib/eval-plan-report.ts new file mode 100644 index 000000000..fef62483f --- /dev/null +++ b/lib/eval-plan-report.ts @@ -0,0 +1,46 @@ +/** + * eval-plan-report: the results check behind `gstack-eval-plan results --check`. + * A preregistration that says `stack: loaded` names its no-gstack arms; its + * results file must hold one markdown table with a filled row for every Arms + * identity, so each no-gstack row sits beside its variants + * (docs/TESTING_INTERNALS.md#no-gstack-arm). Reads text only; executes nothing. + */ +import { armIdentities, checkPrereg } from './eval-plan'; + +export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[] } + +const PLACEHOLDER = /^(?:|-+|—|–|n\/a|tbd|todo|\?)$/i; +const cellsOf = (line: string) => line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(c => c.trim()); +const separator = (line: string | undefined) => !!line && /^\s*\|[\s:|-]+\|?\s*$/.test(line) && line.includes('-'); + +/** The filled-row identities of each markdown table: first cell, backticks stripped, header and separator rows skipped. */ +export function resultTables(text: string): Array> { + const lines = text.split('\n'); + const tables: Array> = []; + let current: Set | null = null; + lines.forEach((line, i) => { + if (!/^\s*\|/.test(line)) { current = null; return; } + if (!current) { current = new Set(); tables.push(current); } + if (separator(line) || separator(lines[i + 1])) return; + const [first, ...rest] = cellsOf(line); + const id = (first ?? '').replace(/`/g, '').trim(); + if (id && rest.some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.add(id); + }); + return tables; +} + +export function checkResults(preregText: string, resultsText: string): ResultsCheck { + const prereg = checkPrereg(preregText); + if (prereg.comparison.stack === 'none') return { ok: true, required: [], no_gstack: [], found: [], missing: [], problems: [] }; + const required = [...new Set(armIdentities(preregText))]; + const noGstack = prereg.no_gstack.arms; + const problems: string[] = []; + if (!prereg.ok) problems.push('the preregistration does not pass prereg --check'); + const tables = resultTables(resultsText); + const best = tables.reduce>((b, t) => (required.filter(id => t.has(id)).length > required.filter(id => b.has(id)).length ? t : b), new Set()); + const found = required.filter(id => best.has(id)); + const missing = required.filter(id => !best.has(id)); + if (!tables.length) problems.push('no markdown table in the results file'); + else for (const id of missing) problems.push(`no filled row for ${id} in the table that holds the other arms`); + return { ok: problems.length === 0, required, no_gstack: noGstack, found, missing, problems }; +} diff --git a/lib/result-codes-eval.ts b/lib/result-codes-eval.ts index 70102382b..3dac8edd2 100644 --- a/lib/result-codes-eval.ts +++ b/lib/result-codes-eval.ts @@ -25,6 +25,11 @@ export const EVAL_RESULT_CODES = { summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own #no-gstack Arms line', fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each #no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line', }, + EVAL_REPORT_NO_GSTACK_ROW: { + anchor: 'eval-report-no-gstack-row', + summary: 'the results file has no single table with a filled row for every arm the preregistration lists, so a no-gstack row is missing or not beside its variants', + fix: 'put one row per arm identity (each #no-gstack beside its variants) in one markdown table of the results file, then rerun `gstack-eval-plan results --check`; publish no number before it says ok', + }, EVAL_ARM_DUPLICATE: { anchor: 'eval-arm-duplicate', summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts', diff --git a/test/eval-plan-results.test.ts b/test/eval-plan-results.test.ts new file mode 100644 index 000000000..1544da203 --- /dev/null +++ b/test/eval-plan-results.test.ts @@ -0,0 +1,96 @@ +/** + * GST-471: `gstack-eval-plan results --check`. A loaded preregistration's + * results file must hold one markdown table with a filled row for every Arms + * identity, so each no-gstack row sits beside its variants. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { checkResults } from '../lib/eval-plan-report'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const dirs: string[] = []; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); + +const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const PREREG = BASE + ` +## Arms +- claude-opus-5-5#review-g default settings +- claude-opus-5-5#no-gstack default settings + +## Comparison +stack: loaded + +## No-gstack arm +arms: claude-opus-5-5#no-gstack +inputs: the same rows +prompt: the task only +panel: rule, 3 trials, same judge +report: results.md, one row beside the variant +`; +const NONE = BASE + '\n## Arms\n- claude-opus-5-5 default\n\n## Comparison\nstack: none\nwhy: scorer calibration\n'; +const table = (...rows: string[]) => ['| arm | passed | cost |', '|---|---|---|', ...rows].join('\n') + '\n'; + +describe('checkResults', () => { + test('one table with a filled row per arm passes; backticks and the header row are handled', () => { + const r = checkResults(PREREG, '# Results\n\n' + table('| `claude-opus-5-5#review-g` | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |')); + expect(r).toMatchObject({ ok: true, no_gstack: ['claude-opus-5-5#no-gstack'], missing: [] }); + }); + + test('a missing no-gstack row fails naming it', () => { + const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |')); + expect(r.ok).toBe(false); + expect(r.problems).toEqual(['no filled row for claude-opus-5-5#no-gstack in the table that holds the other arms']); + }); + + test('placeholder cells do not count as a filled row', () => { + for (const cell of ['', '-', '—', 'n/a', 'TBD']) { + const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', `| claude-opus-5-5#no-gstack | ${cell} | ${cell} |`)); + expect(r.missing, JSON.stringify(cell)).toEqual(['claude-opus-5-5#no-gstack']); + } + }); + + test('the no-gstack row must sit in the same table as its variants; prose mentions never count', () => { + const split = table('| claude-opus-5-5#review-g | 7/9 | $1.20 |') + '\nThe claude-opus-5-5#no-gstack arm passed 5/9.\n\n' + table('| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'); + expect(checkResults(PREREG, split).ok).toBe(false); + expect(checkResults(PREREG, 'claude-opus-5-5#review-g 7/9, claude-opus-5-5#no-gstack 5/9\n').problems).toEqual(['no markdown table in the results file']); + }); + + test('identities match whole: a prefix model does not stand in for the arm', () => { + const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5#no-gstack | 5/9 | $0.80 |')); + expect(r.missing).toEqual(['claude-opus-5-5#no-gstack']); + }); + + test('stack: none needs no row; a preregistration that fails prereg --check fails here too', () => { + expect(checkResults(NONE, 'no table at all\n').ok).toBe(true); + const broken = PREREG.replace(/## No-gstack arm[\s\S]*$/, ''); + const r = checkResults(broken, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |')); + expect(r.problems).toEqual(['the preregistration does not pass prereg --check']); + }); +}); + +describe('gstack-eval-plan results --check', () => { + const run = (prereg: string, results: string) => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-results-check-')); + dirs.push(d); + fs.writeFileSync(path.join(d, 'prereg.md'), prereg); + fs.writeFileSync(path.join(d, 'results.md'), results); + const r = spawnSync(BIN, ['results', '--check', 'prereg.md', 'results.md'], { cwd: d, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; + }; + + test('missing row: RESULTS: incomplete, EVAL_REPORT_NO_GSTACK_ROW, exit 1', () => { + const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |')); + expect(r.code).toBe(1); + expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/1 rows=1/2 file=results.md\n'); + expect(r.err).toContain('(EVAL_REPORT_NO_GSTACK_ROW)'); + }); + + test('complete: RESULTS: ok, exit 0', () => { + const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |')); + expect(r.code, r.err).toBe(0); + expect(r.out).toBe('RESULTS: ok no_gstack_rows=1/1 rows=2/2 file=results.md\n'); + }); +}); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index d56450722..8139bdb26 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -34,6 +34,7 @@ { "file": "lib/design-lint/rules/index.ts", "movedFrom": "(new: the bundled design lint, GSTA-105; ported from hallmark sloplint.mjs)" }, { "file": "lib/design-lint/rules/copy.ts", "movedFrom": "(new: the bundled design lint, GSTA-105; ported from hallmark sloplint.mjs)" }, { "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" }, + { "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" }, { "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" }, { "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" }, { "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" },