From fdf3f1e1b661b65bad6d78d41c006ce51991bc7f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 16 Nov 2026 10:55:19 +0800 Subject: [PATCH] Make /eval-plan check judges, zeros, failure counts and decisions before results count (GST-2754) (#4384) * Make /eval-plan check judges, zeros, failure counts and decisions before results count (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Check zeros in every results table, apply Decisions and Scoring with stack: none, count only valid judge labels (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Describe the /eval-plan result checks in the skills guide (GST-2754) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-doctor-components.sh | 3 + bin/gstack-eval-plan | 50 ++++-- changelog.d/eval-plan-result-checks.md | 1 + docs/TESTING_INTERNALS.md | 31 ++++ docs/skills.md | 9 +- docs/troubleshooting.md | 64 +++++++- eval-plan/SKILL.md | 52 ++++++- eval-plan/SKILL.md.tmpl | 52 ++++++- lib/eval-plan-checks.ts | 121 +++++++++++++++ lib/eval-plan-report.ts | 79 +++++++--- lib/eval-plan.ts | 24 ++- lib/result-codes-eval.ts | 20 ++- test/eval-plan-no-gstack-arm.test.ts | 11 +- test/eval-plan-prereg-diagnostics.test.ts | 8 +- test/eval-plan-result-checks.test.ts | 180 ++++++++++++++++++++++ test/eval-plan-results.test.ts | 3 +- test/exercised-map.test.ts | 1 + test/fixtures/arm-benchmark/PREREG.md | 2 + test/fixtures/model-panel/PREREG.md | 2 + test/helpers/placeholder-allowlist.ts | 1 + test/pricing-eval-plan.test.ts | 4 +- 21 files changed, 643 insertions(+), 75 deletions(-) create mode 100644 changelog.d/eval-plan-result-checks.md create mode 100644 lib/eval-plan-checks.ts create mode 100644 test/eval-plan-result-checks.test.ts diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 4a810df8a..e65205dc0 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -413,6 +413,9 @@ gstack_result_anchor() { PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;; EVAL_REPORT_NO_GSTACK_ROW) printf '%s' 'eval-report-no-gstack-row' ;; + EVAL_JUDGE_UNCHECKED) printf '%s' 'eval-judge-unchecked' ;; + EVAL_REPORT_UNPLANTED_ZERO) printf '%s' 'eval-report-unplanted-zero' ;; + EVAL_REPORT_NO_ANSWER_ONLY) printf '%s' 'eval-report-no-answer-only' ;; EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;; OFFICE_HOURS_PROBE_REFUSED) printf '%s' 'office-hours-probe-refused' ;; TASKS_SUPERSEDE_REFUSED) printf '%s' 'tasks-supersede-refused' ;; diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 4826b83e0..4ea39ba65 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -10,7 +10,8 @@ * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] - * gstack-eval-plan results --check [--json] + * gstack-eval-plan results --check [--labels ] [--json] + * gstack-eval-plan judge --check --labels [--json] * gstack-eval-plan price-gate --gate-file | --gate "" [--plan-file ] [--base ] [--cap-usd N] * [--smoke-cap-usd N] [--allow-unpriced ""] [--out ] [--json] * @@ -27,13 +28,15 @@ * price-gate lint every arm's model against the base branch's * .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete * prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused. - * Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts. + * Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts; the result checks + * (decisions, scoring, judge labels) in lib/eval-plan-checks.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; +import { armModel, checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; import { checkResults } from '../lib/eval-plan-report'; +import { checkJudgeLabels, renderJudgeLabels } from '../lib/eval-plan-checks'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; @@ -43,8 +46,11 @@ import { SpendCapExceeded } from '../lib/spend-ledger'; const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md) prereg --check [--allow-repo-commands] [--json] - the required sections exist and are filled: Bars, Arms, Comparison, Decoys, Stop rules, - Held-out exposure, and No-gstack arm when the Comparison says stack: loaded; + the required sections exist and are filled: Bars, Decisions, Arms, Comparison, Scoring, + Decoys, Stop rules, Held-out exposure, and No-gstack arm when the Comparison says stack: loaded; + Decisions needs two or more "if : " lines with different actions; Scoring a + judge (none, or another model family with labels >= 40 and agreement >= 0.6) and, with two or + more arms, paired: yes, interval: 95% and mdd:; a frozen: date before 2026-11-15 skips both; an optional Diagnostics section's summarizer runs on its synthetic fixture only with --allow-repo-commands (else it is listed and exit 3) and every count must match prereg --init [--title ] write the template @@ -60,9 +66,13 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri settle unknown, and sit outside the cap and the projection until one settles a charge pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + per-arm extrapolation with a 95% band, then the approval ask - results --check [--json] + results --check [--labels labels.jsonl] [--json] one table in the results file has a filled row for every Arms identity, each no-gstack arm - beside its variants; reads text only (docs/TESTING_INTERNALS.md#no-gstack-arm) + beside its variants; a model judge's labels pass; a zero in any results table comes from a + Diagnostics counter with a planted case; a failures column has an answer-only column beside + it; reads text only (docs/TESTING_INTERNALS.md#result-checks) + judge --check --labels labels.jsonl [--json] + Cohen's kappa of the preregistered judge against {"id","human","judge"} hand labels price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a @@ -126,11 +136,11 @@ function doPrereg(rest: string[]): number { const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null; if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n'); else { - process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${PREREG_SECTIONS.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); + process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${r.present.length + r.missing.length + r.empty.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''}${r.grandfathered ? ` frozen=${r.frozen} result_checks=exempt` : ''} file=${file}\n`); if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`); if (run) process.stdout.write(renderPreregDiagnostics(run)); } - const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)]; + const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...r.decisions.errors.map(e => `decisions: ${e}`), ...r.scoring.errors.map(e => `scoring: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)]; if (incomplete.length) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', incomplete.join('; ')) + '\n'); if (!r.no_gstack.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_NO_GSTACK_ARM', `no-gstack arm: ${r.no_gstack.problems.join('; ')}`) + '\n'); if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n'); @@ -141,12 +151,27 @@ function doPrereg(rest: string[]): number { function doResults(rest: string[]): number { const prereg = flag(rest, '--check'); - const results = rest.filter(a => !a.startsWith('--') && a !== prereg)[0]; + const labels = flag(rest, '--labels'); + const results = rest.filter(a => !a.startsWith('--') && a !== prereg && a !== labels)[0]; if (!prereg || !results || !fs.existsSync(prereg) || !fs.existsSync(results)) return usage('results needs --check '); - const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8')); + if (labels !== undefined && !fs.existsSync(labels)) return usage('results --labels needs an existing labels file'); + const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8'), labels === undefined ? null : fs.readFileSync(labels, 'utf8')); if (rest.includes('--json')) process.stdout.write(JSON.stringify({ prereg, results, ...r }) + '\n'); else process.stdout.write(`RESULTS: ${r.ok ? 'ok' : 'incomplete'} no_gstack_rows=${r.no_gstack.filter(id => r.found.includes(id)).length}/${r.no_gstack.length} rows=${r.found.length}/${r.required.length} file=${results}\n`); - if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan results', 'EVAL_REPORT_NO_GSTACK_ROW', r.problems.join('; ')) + '\n'); return EXIT.fail; } + if (r.judge && !rest.includes('--json')) process.stdout.write(renderJudgeLabels(r.judge)); + for (const code of new Set(r.issues.map(i => i.code))) process.stderr.write(resultLine('gstack-eval-plan results', code, r.issues.filter(i => i.code === code).map(i => i.message).join('; ')) + '\n'); + return r.ok ? EXIT.ok : EXIT.fail; +} + +function doJudge(rest: string[]): number { + const prereg = flag(rest, '--check'); + const labels = flag(rest, '--labels'); + if (!prereg || !labels || !fs.existsSync(prereg) || !fs.existsSync(labels)) return usage('judge needs --check --labels '); + const { scoring } = checkPrereg(fs.readFileSync(prereg, 'utf8')); + if (!scoring.judge || scoring.judge.toLowerCase() === 'none') return usage('judge --check needs a preregistration whose ## Scoring names a judge model'); + const r = checkJudgeLabels(scoring, fs.readFileSync(labels, 'utf8')); + process.stdout.write(rest.includes('--json') ? JSON.stringify({ prereg, labels, ...r }) + '\n' : renderJudgeLabels(r)); + if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan judge', 'EVAL_JUDGE_UNCHECKED', r.problems.join('; ')) + '\n'); return EXIT.fail; } return EXIT.ok; } @@ -266,6 +291,7 @@ function main(argv: string[]): number { if (cmd === 'pilot') return doPilot(rest); if (cmd === 'price-gate') return doPriceGate(rest); if (cmd === 'results') return doResults(rest); + if (cmd === 'judge') return doJudge(rest); return usage(`unknown command ${cmd}`); } catch (e: any) { if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; } diff --git a/changelog.d/eval-plan-result-checks.md b/changelog.d/eval-plan-result-checks.md new file mode 100644 index 000000000..8613ccc0a --- /dev/null +++ b/changelog.d/eval-plan-result-checks.md @@ -0,0 +1 @@ +- `/eval-plan` now runs four result checks instead of leaving them to judgment. `prereg --check` requires a `## Decisions` section (what each possible result changes, at least two different actions) and a `## Scoring` section (a judge from another model family than every arm, checked against at least 40 hand labels at kappa 0.6 or better, and paired results with a 95% interval and the smallest detectable difference stated before the run). The new `judge --check --labels` scores the judge against hand labels, and `results --check` holds back a judged result without passing labels, a zero from a counter never shown to find a planted case, and a failures column with no answer-only column beside it. Preregistrations frozen before 2026-11-15 keep their rules. diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index c4abc8f9f..65042c1da 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -2059,6 +2059,37 @@ it as the diagnostic. Candidate folders and git identity name a project, never the arm. The model panel (`test/skill-e2e-model-panel.test.ts`) runs a `#no-gstack` arm beside each model (GST-468, GST-469). +## Result checks + +`/eval-plan` runs four checks so that a number which cannot change a decision, or +which comes from an unchecked instrument, never leaves the run. Each one is a +refusal from `bin/gstack-eval-plan` (logic in `lib/eval-plan-checks.ts` and +`lib/eval-plan-report.ts`), not advice: + +| Check | When it runs | Refusal | +|---|---|---| +| `## Decisions`: one `if : ` line per possible result, at least two, with different actions | `prereg --check`, before anything is spent | `PREREG_INCOMPLETE` | +| `## Scoring`: `judge: none`, or a judge model from another family than every arm with `labels:` (40 or more) and `agreement:` (kappa 0.6 or more); with two or more arms, `paired: yes`, `interval: 95%` and `mdd:` stated before the run | `prereg --check` | `PREREG_INCOMPLETE` | +| The judge's Cohen's kappa against `{"id", "human", "judge"}` hand labels, at the preregistered count and bar | `judge --check` before the approval ask; `results --check --labels` before any number goes out | `EVAL_JUDGE_UNCHECKED` | +| A zero in any results table comes from a column named after a `## Diagnostics` counter whose synthetic fixture plants at least one case (expected 1 or more) | `results --check` | `EVAL_REPORT_UNPLANTED_ZERO` | +| A results table with a failures column also has an `answer-only` column: failures by the grader's own rule on the final answer | `results --check` | `EVAL_REPORT_NO_ANSWER_ONLY` | + +Why: in our own agent teams, a counter that never fired reported zero for one +model while an outside check with planted cases found every one, and a reported +rise in failures vanished once failures were counted by the grader's own rule. +A judge from the same family as the arm it grades, or one never compared with +people, gives numbers nobody can weigh. An eval whose every result leads to the +same action spends money to change nothing. + +Model families come from the model id (`claude` is Anthropic, `gpt` and `o` +models OpenAI, `gemini` Google, `grok` xAI), else its first word. A judge +whose labels are all one class has undefined agreement and is refused. + +A preregistration with a `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the +rules it was frozen under: the PREREG line shows `result_checks=exempt` and +none of the four checks apply. Text frozen before a priced call is not edited +to meet a newer rule. + ## Frontier models under test Garry, 2026-11-01 (R54): "I think you are spending too much time with Haiku and diff --git a/docs/skills.md b/docs/skills.md index e45ad381c..8e2fc459f 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -966,7 +966,10 @@ You: /eval-plan how much will the retrieval eval cost before I run it? `bin/gstack-eval-plan prereg --init` writes the preregistration (question, bars, arms, decoys, stop rules, cap) and `--check` refuses to continue while a field is -blank (`PREREG_INCOMPLETE`). `arms` resolves the `eval-arms` model role against +blank (`PREREG_INCOMPLETE`), including the `## Decisions` and `## Scoring` +sections the [result checks](TESTING_INTERNALS.md#result-checks) require. +`judge --check --labels ` scores a model judge against hand +labels (`EVAL_JUDGE_UNCHECKED`). `arms` resolves the `eval-arms` model role against `lib/pricing.ts` and stops on an unpriced model (`EVAL_PRICE_MISSING`); `dry-run` runs every row through the stub model for $0 and flags malformed rows (`EVAL_DRY_RUN_FAILED`), and refuses a plan whose every arm is Haiku or Sonnet @@ -981,7 +984,9 @@ stays out of the projection until one of its attempts settles a real charge. A run stops at the first quota or rate-limit error on such an arm. `results --check ` refuses a finished results file unless one table holds a filled row for every arm, each `#no-gstack` row beside its -variants (`EVAL_REPORT_NO_GSTACK_ROW`); no number leaves the run before it passes. +variants (`EVAL_REPORT_NO_GSTACK_ROW`); it also checks a model judge's `--labels`, +unplanted zeros (`EVAL_REPORT_UNPLANTED_ZERO`) and failures columns without an +`answer-only` column (`EVAL_REPORT_NO_ANSWER_ONLY`); no number leaves the run before it passes. The skill's closing section lists the tier-3 items (instrument audit, steady state, sealed paths, off-machine persistence, memory preflight, power tables, QA soak) as `tier 3, after P7`; none of them run yet. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f14f21b3d..50c43ef6a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -4763,12 +4763,19 @@ parentheses; grep the code. ### `PREREG_INCOMPLETE` **Meaning.** `gstack-eval-plan prereg --check ` found a required -preregistration section missing or empty: `## Bars`, `## Arms`, -`## Comparison`, `## Decoys`, `## Stop rules` or `## Held-out exposure` -(every held-out set the plan touches, its prior exposure and the reservation -on it). It also fires when `## Comparison` has no `stack:` line, a value other -than `loaded` or `none`, or `stack: none` without a `why:` line, and when a -`## Diagnostics` declaration is malformed. +preregistration section missing or empty: `## Bars`, `## Decisions`, +`## Arms`, `## Comparison`, `## Scoring`, `## Decoys`, `## Stop rules` or +`## Held-out exposure` (every held-out set the plan touches, its prior +exposure and the reservation on it). It also fires when `## Comparison` has no +`stack:` line, a value other than `loaded` or `none`, or `stack: none` without +a `why:` line, and when a `## Diagnostics` declaration is malformed. The +result checks add two more (`decisions:` and `scoring:` in the message): +`## Decisions` needs two or more `if : ` lines with different +actions, and `## Scoring` needs a `judge:` line, a model judge from another +model family than every arm with `labels:` of 40 or more and `agreement:` of +0.6 or more, and with two or more arms `paired: yes`, `interval: 95%` and +`mdd:`. A `frozen:` date before 2026-11-15 exempts both +([docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks)). **Fix.** Fill the named sections before any priced call. A plan that cannot name its stop rule or its held-out exposure is not preregistered; the pilot @@ -4808,6 +4815,51 @@ in one table, and rerun the check. Publish no number before it prints `RESULTS: ok`. A preregistration with `stack: none` needs no row. The rule is in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm). + +### `EVAL_JUDGE_UNCHECKED` + +**Meaning.** The preregistration's `## Scoring` names a model judge, and +`gstack-eval-plan judge --check` or `results --check` found no `--labels` +file, fewer labelled outputs than the preregistered count (at least 40), labels +that are all one class (agreement is undefined), a repeated id, or a Cohen's +kappa between the hand labels and the judge below the preregistered bar (at +least 0.6). Until it passes, the judge's numbers do not count. + +**Fix.** Label outputs by hand from development or calibration rows, never +held-out ones, as JSONL rows `{"id", "human", "judge"}`, covering every kind +of output the judge scores, and pass the file with `--labels`. A judge below +its bar is replaced or its prompt fixed and relabelled; the bar is not +lowered. The rule is in +[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks). + + +### `EVAL_REPORT_UNPLANTED_ZERO` + +**Meaning.** `gstack-eval-plan results --check` found a zero (`0`, `0%`, +`0/N`) in a column of any results table, and the preregistration's +`## Diagnostics` declares no counter by that column's name with a planted +case (`expected` of 1 or more). A counter that never fires also reports zero, +so a zero from a counter nobody saw find anything proves nothing. + +**Fix.** Declare the counter under `## Diagnostics` with a synthetic fixture +that plants at least one case, run `prereg --check --allow-repo-commands` +until its `PREREG_DIAGNOSTIC:` line says ok, and name the results column after +the diagnostic. Otherwise leave the zero out of the report. The rule is in +[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks). + + +### `EVAL_REPORT_NO_ANSWER_ONLY` + +**Meaning.** A table in the results file has a column counting failures and no +column headed `answer-only` beside it. A broad failure count (from +transcripts, tool errors or a second rule) can rise while failures counted by +the grader's own rule on the final answer do not move, and the two readings +must be read side by side. + +**Fix.** Add an `answer-only` column to the same table, counting failures by +the grader's rule on the final answer alone, and report both readings. The +rule is in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks). + ### `EVAL_ARM_DUPLICATE` diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index adca9139c..9f7d7ea02 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -134,9 +134,11 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| -| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction, what a miss means for the decision | +| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction | +| Decisions | one `if : ` line per way the run can come out, at least two, with different actions | | Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | +| Scoring | `judge: none` or the judge model with `labels:` and `agreement:`; with two or more arms, `paired: yes`, `interval: 95%`, `mdd:` | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | @@ -169,7 +171,22 @@ fixture is built to produce. The summarizer's last stdout line is one JSON objec Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`. `PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an -undeclared one does not match its definition. Fix that before you freeze. +undeclared one does not match its definition. Fix that before you freeze. Any count +the results may report as zero is declared here with a planted case: its fixture holds +at least one, so `expected` is 1 or more and the summarizer is shown to find it. + +**Result checks** (docs/TESTING_INTERNALS.md#result-checks). `prereg --check` runs them; +`PREREG_INCOMPLETE` names the line that fails: + +1. **Decisions before spend.** Each possible result names what we do differently. If + every result leads to the same action, the eval changes nothing: do not run it. +2. **A checked judge.** A model judge comes from another model family than every arm, + is checked against at least 40 hand-labelled outputs (never held-out rows) at a + kappa of 0.6 or better, and the comparison is paired with a 95% interval and its + smallest detectable difference stated now. + +A `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the rules it was frozen under +(`result_checks=exempt`). ### Step 1b: Arms and their price @@ -284,6 +301,18 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. ## Step 4: The approval ask +With a model judge, check it before asking. Label at least the preregistered number of +outputs by hand as `{"id", "human", "judge"}` JSONL (development or calibration rows), +then: + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan judge --check --labels +``` + +`JUDGE_AGREEMENT: ok` continues. `EVAL_JUDGE_UNCHECKED` (exit 1) means too few labels, +one class, or kappa below the bar: replace the judge or fix its prompt, then +relabel; never lower the bar. + Price every paid gate the plan names (a benchmark or eval command that spends) from the repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack). The gate command is free text, so it reaches the pricer only as a file: @@ -309,7 +338,8 @@ Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STAL `--allow-unpriced ""` only on the owner's word. Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend -table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one +table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, the `JUDGE_AGREEMENT:` +line, the preregistration's Decisions lines, and its stop rules. Then ask one question with the projection as the recommended option only when `verdict=within`: - A) Run `` items at the projected cost (recommended when within the cap) @@ -328,12 +358,20 @@ from the Arms section, each `#no-gstack` row beside its variants. Before leaves the run (a report, a READY block, a PR body), check it: ```bash -~/.claude/skills/gstack/bin/gstack-eval-plan results --check +~/.claude/skills/gstack/bin/gstack-eval-plan results --check [--labels ] ``` -`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm -whose filled row is missing from the table that holds the others; add it and rerun. A -preregistration with `stack: none` needs no row. +`RESULTS: ok` lets the numbers go out. Each refusal exits 1: + +- `EVAL_REPORT_NO_GSTACK_ROW` names the arm whose filled row is missing from the table + that holds the others; add it. A preregistration with `stack: none` needs no row. +- `EVAL_JUDGE_UNCHECKED`: a model judge's numbers count only with `--labels` passing. +- `EVAL_REPORT_UNPLANTED_ZERO`: a zero in any results table, in a column not named after + a Diagnostics counter with a planted case. Declare and run it, or report the zero as + unverified, outside the tables. +- `EVAL_REPORT_NO_ANSWER_ONLY`: a failures column needs an `answer-only` column beside + it (failures by the grader's own rule on the final answer); a broader count can rise + while that one stays flat. The results file opens with two or three plain takeaways ("We learned X. You should do X."): what works first, then what didn't, with its number. Cost appears only as part of a diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 4472766b0..c51321370 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -56,9 +56,11 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| -| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction, what a miss means for the decision | +| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction | +| Decisions | one `if : ` line per way the run can come out, at least two, with different actions | | Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | | Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | +| Scoring | `judge: none` or the judge model with `labels:` and `agreement:`; with two or more arms, `paired: yes`, `interval: 95%`, `mdd:` | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | @@ -91,7 +93,22 @@ fixture is built to produce. The summarizer's last stdout line is one JSON objec Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`. `PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an -undeclared one does not match its definition. Fix that before you freeze. +undeclared one does not match its definition. Fix that before you freeze. Any count +the results may report as zero is declared here with a planted case: its fixture holds +at least one, so `expected` is 1 or more and the summarizer is shown to find it. + +**Result checks** (docs/TESTING_INTERNALS.md#result-checks). `prereg --check` runs them; +`PREREG_INCOMPLETE` names the line that fails: + +1. **Decisions before spend.** Each possible result names what we do differently. If + every result leads to the same action, the eval changes nothing: do not run it. +2. **A checked judge.** A model judge comes from another model family than every arm, + is checked against at least 40 hand-labelled outputs (never held-out rows) at a + kappa of 0.6 or better, and the comparison is paired with a 95% interval and its + smallest detectable difference stated now. + +A `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the rules it was frozen under +(`result_checks=exempt`). ### Step 1b: Arms and their price @@ -206,6 +223,18 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`. ## Step 4: The approval ask +With a model judge, check it before asking. Label at least the preregistered number of +outputs by hand as `{"id", "human", "judge"}` JSONL (development or calibration rows), +then: + +```bash +~/.claude/skills/gstack/bin/gstack-eval-plan judge --check --labels +``` + +`JUDGE_AGREEMENT: ok` continues. `EVAL_JUDGE_UNCHECKED` (exit 1) means too few labels, +one class, or kappa below the bar: replace the judge or fix its prompt, then +relabel; never lower the bar. + Price every paid gate the plan names (a benchmark or eval command that spends) from the repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack). The gate command is free text, so it reaches the pricer only as a file: @@ -224,7 +253,8 @@ Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STAL `--allow-unpriced ""` only on the owner's word. Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend -table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one +table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, the `JUDGE_AGREEMENT:` +line, the preregistration's Decisions lines, and its stop rules. Then ask one question with the projection as the recommended option only when `verdict=within`: - A) Run `` items at the projected cost (recommended when within the cap) @@ -243,12 +273,20 @@ from the Arms section, each `#no-gstack` row beside its variants. Before leaves the run (a report, a READY block, a PR body), check it: ```bash -~/.claude/skills/gstack/bin/gstack-eval-plan results --check +~/.claude/skills/gstack/bin/gstack-eval-plan results --check [--labels ] ``` -`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm -whose filled row is missing from the table that holds the others; add it and rerun. A -preregistration with `stack: none` needs no row. +`RESULTS: ok` lets the numbers go out. Each refusal exits 1: + +- `EVAL_REPORT_NO_GSTACK_ROW` names the arm whose filled row is missing from the table + that holds the others; add it. A preregistration with `stack: none` needs no row. +- `EVAL_JUDGE_UNCHECKED`: a model judge's numbers count only with `--labels` passing. +- `EVAL_REPORT_UNPLANTED_ZERO`: a zero in any results table, in a column not named after + a Diagnostics counter with a planted case. Declare and run it, or report the zero as + unverified, outside the tables. +- `EVAL_REPORT_NO_ANSWER_ONLY`: a failures column needs an `answer-only` column beside + it (failures by the grader's own rule on the final answer); a broader count can rise + while that one stays flat. The results file opens with two or three plain takeaways ("We learned X. You should do X."): what works first, then what didn't, with its number. Cost appears only as part of a diff --git a/lib/eval-plan-checks.ts b/lib/eval-plan-checks.ts new file mode 100644 index 000000000..4491fc396 --- /dev/null +++ b/lib/eval-plan-checks.ts @@ -0,0 +1,121 @@ +/** + * eval-plan-checks: the result checks /eval-plan runs on top of the section + * check (docs/TESTING_INTERNALS.md#result-checks). `## Decisions` names what + * each possible result changes before anything is spent; `## Scoring` names + * the judge and the statistics before the run; `judge --check` scores the + * judge against hand labels before its numbers count. A preregistration whose + * `frozen:` date is before RESULT_CHECKS_SINCE keeps the rules it was frozen + * under. Reads text only; executes nothing. + */ +export const RESULT_CHECKS_SINCE = '2026-11-15'; +export const JUDGE_MIN_LABELS = 40; +export const JUDGE_MIN_KAPPA = 0.6; + +const PLACEHOLDER = /^(?:|-+|—|n\/a|none|tbd|todo|\?)\.?$/i; +const keyValues = (body: string[]) => { + const kv: Record = {}; + for (const line of body) { + const m = /^\s*(?:[-*]\s+)?([a-z]+):(.*)$/i.exec(line); + if (m) kv[m[1]!.toLowerCase()] = m[2]!.trim().replace(/^`(.*)`$/, '$1'); + } + return kv; +}; +const firstNumber = (v: string | undefined) => { const m = /\d+(?:\.\d+)?/.exec(v ?? ''); return m ? Number(m[0]) : null; }; + +/** The `frozen: YYYY-MM-DD` line, when the preregistration states its freeze date. */ +export function frozenDate(text: string): string | null { + return /^\s*frozen:\s*(\d{4}-\d{2}-\d{2})\s*$/im.exec(text)?.[1] ?? null; +} +export const grandfathered = (text: string) => { const d = frozenDate(text); return d !== null && d < RESULT_CHECKS_SINCE; }; + +/** Model family for the judge rule: the lab behind a model id, else the id's first word. */ +export function modelFamily(model: string): string { + const id = model.split('#')[0]!.toLowerCase().split('/').pop()!; + const head = id.split(/[-_.:\s]|(?<=[a-z])(?=\d)/)[0]!; + const labs: Record = { claude: 'anthropic', gpt: 'openai', o: 'openai', codex: 'openai', gemini: 'google', gemma: 'google', grok: 'xai', llama: 'meta', muse: 'meta' }; + return labs[head] ?? head; +} + +export interface DecisionsCheck { outcomes: Array<{ result: string; action: string }>; errors: string[] } +/** `## Decisions`: one `if : ` line per possible result, at least two, and not all the same action. */ +export function checkDecisions(body: string[] | null): DecisionsCheck { + const outcomes: DecisionsCheck['outcomes'] = []; + for (const line of body ?? []) { + const m = /^\s*(?:[-*]\s+)?if\s+(.+?):\s*(.*)$/i.exec(line); + if (m) outcomes.push({ result: m[1]!.trim(), action: m[2]!.trim() }); + } + if (!body?.length) return { outcomes, errors: [] }; + const errors: string[] = []; + if (outcomes.length < 2) errors.push('needs one "if : " line per possible result, at least two'); + for (const o of outcomes) if (PLACEHOLDER.test(o.action)) errors.push(`if ${o.result}: names no action`); + const actions = new Set(outcomes.map(o => o.action.toLowerCase().replace(/\s+/g, ' '))); + if (outcomes.length >= 2 && actions.size === 1) errors.push('every result leads to the same action, so the eval changes nothing; do not run it'); + return { outcomes, errors }; +} + +export interface ScoringCheck { judge: string | null; labels: number | null; agreement: number | null; errors: string[] } +/** + * `## Scoring`: `judge: none` or the judge model; a model judge needs `labels:` + * (at least 40) and `agreement:` (a kappa bar of at least 0.6) and a family no + * arm shares. Two or more arms need `paired: yes`, `interval: 95%` and `mdd:`, + * the smallest difference the run can detect. + */ +export function checkScoring(body: string[] | null, arms: string[]): ScoringCheck { + if (!body?.length) return { judge: null, labels: null, agreement: null, errors: [] }; + const kv = keyValues(body); + const judge = kv.judge || null; + const labels = firstNumber(kv.labels); + const agreement = firstNumber(kv.agreement); + const errors: string[] = []; + if (!judge) errors.push('no judge: line (judge: none, or the judge model)'); + else if (judge.toLowerCase() !== 'none') { + if (labels === null || labels < JUDGE_MIN_LABELS) errors.push(`labels: needs at least ${JUDGE_MIN_LABELS} hand-labelled outputs the judge is checked against`); + if (agreement === null || agreement < JUDGE_MIN_KAPPA || agreement > 1) errors.push(`agreement: needs a kappa bar from ${JUDGE_MIN_KAPPA} to 1`); + const family = modelFamily(judge); + for (const arm of [...new Set(arms)]) if (modelFamily(arm) === family) errors.push(`judge ${judge} is the same model family (${family}) as arm ${arm}; use a judge from another family`); + } + if (new Set(arms).size >= 2) { + if (!/^yes\b/i.test(kv.paired ?? '')) errors.push('paired: yes (every comparison is reported on paired items)'); + if (firstNumber(kv.interval) !== 95) errors.push('interval: 95% (the interval every comparison is reported with)'); + if (PLACEHOLDER.test(kv.mdd ?? '')) errors.push('mdd: the smallest difference this run can detect, stated before the run'); + } + return { judge, labels, agreement, errors }; +} + +export interface JudgeLabelCheck { ok: boolean; judge: string; n: number; kappa: number | null; need_n: number; need_kappa: number; problems: string[] } + +/** Cohen's kappa over paired categorical labels; null when chance agreement is total (every label one class). */ +export function cohensKappa(pairs: Array<[string, string]>): number | null { + if (!pairs.length) return null; + const n = pairs.length; + const a = new Map(); const b = new Map(); + let agree = 0; + for (const [h, j] of pairs) { a.set(h, (a.get(h) ?? 0) + 1); b.set(j, (b.get(j) ?? 0) + 1); if (h === j) agree++; } + const pe = [...a.keys()].reduce((s, k) => s + (a.get(k)! / n) * ((b.get(k) ?? 0) / n), 0); + return pe === 1 ? null : (agree / n - pe) / (1 - pe); +} + +/** Score the preregistered judge against a JSONL file of `{"id", "human", "judge"}` rows. */ +export function checkJudgeLabels(scoring: ScoringCheck, labelsText: string): JudgeLabelCheck { + const need_n = Math.max(JUDGE_MIN_LABELS, scoring.labels ?? 0); + const need_kappa = Math.max(JUDGE_MIN_KAPPA, scoring.agreement ?? 0); + const problems: string[] = []; + const pairs: Array<[string, string]> = []; + const ids = new Set(); + labelsText.split('\n').forEach((line, i) => { + if (!line.trim()) return; + let row: { id?: unknown; human?: unknown; judge?: unknown } | null = null; + try { row = JSON.parse(line); } catch { row = null; } + if (!row || typeof row !== 'object' || ![row.id, row.human, row.judge].every(v => (typeof v === 'string' && v.trim() !== '') || typeof v === 'number' || typeof v === 'boolean')) { problems.push(`line ${i + 1}: needs {"id", "human", "judge"}, each a string, number or boolean`); return; } + if (ids.has(String(row.id))) { problems.push(`id ${String(row.id)} is labelled twice`); return; } + ids.add(String(row.id)); + pairs.push([String(row.human), String(row.judge)]); + }); + const kappa = cohensKappa(pairs); + if (pairs.length < need_n) problems.push(`${pairs.length} labelled outputs, needs ${need_n}`); + if (pairs.length && kappa === null) problems.push('every label is one class, so agreement is undefined; label outputs of every kind the judge scores'); + else if (kappa !== null && kappa < need_kappa) problems.push(`kappa ${kappa.toFixed(2)} is below the bar ${need_kappa}`); + return { ok: problems.length === 0, judge: scoring.judge ?? 'none', n: pairs.length, kappa, need_n, need_kappa, problems }; +} + +export const renderJudgeLabels = (c: JudgeLabelCheck) => `JUDGE_AGREEMENT: ${c.ok ? 'ok' : 'unchecked'} judge=${c.judge} n=${c.n}/${c.need_n} kappa=${c.kappa === null ? 'undefined' : c.kappa.toFixed(2)} bar=${c.need_kappa}\n`; diff --git a/lib/eval-plan-report.ts b/lib/eval-plan-report.ts index fef62483f..beabde886 100644 --- a/lib/eval-plan-report.ts +++ b/lib/eval-plan-report.ts @@ -3,44 +3,77 @@ * A preregistration that says `stack: loaded` names its no-gstack arms; its * results file must hold one markdown table with a filled row for every Arms * identity, so each no-gstack row sits beside its variants - * (docs/TESTING_INTERNALS.md#no-gstack-arm). Reads text only; executes nothing. + * (docs/TESTING_INTERNALS.md#no-gstack-arm). The result checks + * (docs/TESTING_INTERNALS.md#result-checks) hold back a model judge's numbers + * until its hand labels pass, a zero from a counter never shown to find a + * planted case, and a failures column with no answer-only reading beside it. + * Reads text only; executes nothing. */ import { armIdentities, checkPrereg } from './eval-plan'; +import { checkJudgeLabels, type JudgeLabelCheck } from './eval-plan-checks'; +import type { ResultCodeName } from './result-codes'; -export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[] } +export interface ResultsIssue { code: ResultCodeName; message: string } +export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[]; issues: ResultsIssue[]; judge: JudgeLabelCheck | null } +export interface ResultTable { ids: Set; header: string[]; rows: string[][] } const PLACEHOLDER = /^(?:|-+|—|–|n\/a|tbd|todo|\?)$/i; +const ZERO = /^0(?:\.0+)?(?:%|\/\d+)?$/; +const plain = (c: string) => c.replace(/[`*]/g, '').trim(); const cellsOf = (line: string) => line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(c => c.trim()); const separator = (line: string | undefined) => !!line && /^\s*\|[\s:|-]+\|?\s*$/.test(line) && line.includes('-'); -/** The filled-row identities of each markdown table: first cell, backticks stripped, header and separator rows skipped. */ -export function resultTables(text: string): Array> { +/** Each markdown table: its header cells, data rows, and filled-row identities (first cell, backticks stripped). */ +export function resultTables(text: string): ResultTable[] { const lines = text.split('\n'); - const tables: Array> = []; - let current: Set | null = null; + const tables: ResultTable[] = []; + let current: ResultTable | null = null; lines.forEach((line, i) => { if (!/^\s*\|/.test(line)) { current = null; return; } - if (!current) { current = new Set(); tables.push(current); } - if (separator(line) || separator(lines[i + 1])) return; - const [first, ...rest] = cellsOf(line); - const id = (first ?? '').replace(/`/g, '').trim(); - if (id && rest.some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.add(id); + if (!current) { current = { ids: new Set(), header: [], rows: [] }; tables.push(current); } + if (separator(line)) return; + if (separator(lines[i + 1])) { current.header = cellsOf(line).map(plain); return; } + const cells = cellsOf(line); + current.rows.push(cells.map(plain)); + const id = (cells[0] ?? '').replace(/`/g, '').trim(); + if (id && cells.slice(1).some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.ids.add(id); }); return tables; } -export function checkResults(preregText: string, resultsText: string): ResultsCheck { +export function checkResults(preregText: string, resultsText: string, labelsText: string | null = null): ResultsCheck { const prereg = checkPrereg(preregText); - if (prereg.comparison.stack === 'none') return { ok: true, required: [], no_gstack: [], found: [], missing: [], problems: [] }; - const required = [...new Set(armIdentities(preregText))]; - const noGstack = prereg.no_gstack.arms; - const problems: string[] = []; - if (!prereg.ok) problems.push('the preregistration does not pass prereg --check'); + const issues: ResultsIssue[] = []; const tables = resultTables(resultsText); - const best = tables.reduce>((b, t) => (required.filter(id => t.has(id)).length > required.filter(id => b.has(id)).length ? t : b), new Set()); - const found = required.filter(id => best.has(id)); - const missing = required.filter(id => !best.has(id)); - if (!tables.length) problems.push('no markdown table in the results file'); - else for (const id of missing) problems.push(`no filled row for ${id} in the table that holds the other arms`); - return { ok: problems.length === 0, required, no_gstack: noGstack, found, missing, problems }; + const arms = [...new Set(armIdentities(preregText))]; + const required = prereg.comparison.stack === 'none' ? [] : arms; + const best = tables.reduce((b, t) => (!b || arms.filter(id => t.ids.has(id)).length > arms.filter(id => b.ids.has(id)).length ? t : b), null); + const found = required.filter(id => best?.ids.has(id)); + const missing = required.filter(id => !best?.ids.has(id)); + if (prereg.comparison.stack !== 'none') { + if (!prereg.ok) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: 'the preregistration does not pass prereg --check' }); + if (!tables.length) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: 'no markdown table in the results file' }); + else for (const id of missing) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: `no filled row for ${id} in the table that holds the other arms` }); + } + let judge: JudgeLabelCheck | null = null; + if (!prereg.grandfathered) { + for (const [section, errors] of [['Decisions', prereg.decisions.errors], ['Scoring', prereg.scoring.errors]] as const) { + for (const e of errors) issues.push({ code: 'PREREG_INCOMPLETE', message: `## ${section}: ${e}` }); + } + if (prereg.scoring.judge && prereg.scoring.judge.toLowerCase() !== 'none') { + judge = labelsText === null ? null : checkJudgeLabels(prereg.scoring, labelsText); + if (!judge) issues.push({ code: 'EVAL_JUDGE_UNCHECKED', message: `the preregistration names judge ${prereg.scoring.judge}; pass --labels ` }); + else if (!judge.ok) issues.push({ code: 'EVAL_JUDGE_UNCHECKED', message: judge.problems.join('; ') }); + } + const planted = new Set((prereg.diagnostics?.expected ?? []).filter(e => e.count >= 1).map(e => e.name.toLowerCase())); + for (const t of tables) { + t.header.forEach((name, col) => { + if (col === 0 || planted.has(name.toLowerCase()) || !t.rows.some(r => ZERO.test(r[col] ?? ''))) return; + issues.push({ code: 'EVAL_REPORT_UNPLANTED_ZERO', message: `column "${name}" reports a zero, and no Diagnostics counter by that name was shown to find a planted case` }); + }); + const failures = t.header.filter(h => /\bfail/i.test(h) && !/answer[- ]only/i.test(h)); + if (failures.length && !t.header.some(h => /answer[- ]only/i.test(h))) issues.push({ code: 'EVAL_REPORT_NO_ANSWER_ONLY', message: `column "${failures[0]}" counts failures with no answer-only column beside it in the same table` }); + } + } + return { ok: issues.length === 0, required, no_gstack: prereg.no_gstack.arms, found, missing, problems: issues.map(i => i.message), issues, judge }; } diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 258ee975a..9b5580c1e 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -19,16 +19,19 @@ import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pric import type { ResultCodeName } from './result-codes'; import type { ModelRulesCheck } from './eval-models'; import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger'; +import { checkDecisions, checkScoring, frozenDate, grandfathered, type DecisionsCheck, type ScoringCheck } from './eval-plan-checks'; // --------------------------------------------------------------------------- // preregistration // --------------------------------------------------------------------------- -export const PREREG_SECTIONS = ['Bars', 'Arms', 'Comparison', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; +export const PREREG_SECTIONS = ['Bars', 'Decisions', 'Arms', 'Comparison', 'Scoring', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; +/** Sections added with the result checks; a preregistration frozen before them (`frozen:` before RESULT_CHECKS_SINCE) is not asked for them. */ +const RESULT_CHECK_SECTIONS: readonly string[] = ['Decisions', 'Scoring']; /** `## Comparison`: `stack: loaded` when any arm loads gstack or another skill stack, `stack: none` with a `why:` otherwise. */ export interface ComparisonCheck { stack: 'loaded' | 'none' | null; why: string | null; errors: string[] } /** `## No-gstack arm`, required when `stack: loaded`: one `#no-gstack` Arms line per model, the inputs, prompt, panel and report it shares with the variants, and its masking, install and audit lines. */ export interface NoGstackArmCheck { required: boolean; ok: boolean; arms: string[]; problems: string[] } -export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null } +export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null; decisions: DecisionsCheck; scoring: ScoringCheck; frozen: string | null; grandfathered: boolean } export const NO_GSTACK_LOADOUT = 'no-gstack'; /** masking, install and audit keep the arm clean (TESTING_INTERNALS.md#no-gstack-arm): an arm that read a gstack skill file is contaminated. */ export const NO_GSTACK_KEYS = ['arms', 'inputs', 'prompt', 'panel', 'report', 'masking', 'install', 'audit'] as const; @@ -37,7 +40,10 @@ export function preregTemplate(title = 'Eval preregistration'): string { return `# ${title} ## Bars - + + +## Decisions + ## Arms @@ -48,6 +54,9 @@ export function preregTemplate(title = 'Eval preregistration'): string { ## No-gstack arm +## Scoring + + ## Decoys @@ -56,7 +65,7 @@ export function preregTemplate(title = 'Eval preregistration'): string { ## Held-out exposure - + `; } @@ -130,14 +139,19 @@ function checkNoGstackArm(text: string, stack: ComparisonCheck['stack']): NoGsta export function checkPrereg(text: string): PreregCheck { const present: string[] = []; const missing: string[] = []; const empty: string[] = []; + const exempt = grandfathered(text); for (const name of PREREG_SECTIONS) { + if (exempt && RESULT_CHECK_SECTIONS.includes(name)) continue; const body = sectionBody(text, name); if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name); } const comparison = checkComparison(text); const no_gstack = checkNoGstackArm(text, comparison.stack); const diagnostics = parsePreregDiagnostics(text); - return { ok: missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length, present, missing, empty, comparison, no_gstack, diagnostics }; + const decisions = exempt ? { outcomes: [], errors: [] } : checkDecisions(sectionBody(text, 'Decisions')); + const scoring = exempt ? { judge: null, labels: null, agreement: null, errors: [] } : checkScoring(sectionBody(text, 'Scoring'), armIdentities(text)); + const ok = missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length && !decisions.errors.length && !scoring.errors.length; + return { ok, present, missing, empty, comparison, no_gstack, diagnostics, decisions, scoring, frozen: frozenDate(text), grandfathered: exempt }; } // --------------------------------------------------------------------------- diff --git a/lib/result-codes-eval.ts b/lib/result-codes-eval.ts index e019e3bdd..23d6143cf 100644 --- a/lib/result-codes-eval.ts +++ b/lib/result-codes-eval.ts @@ -10,8 +10,8 @@ import type { ResultCode } from './result-codes'; export const EVAL_RESULT_CODES = { PREREG_INCOMPLETE: { anchor: 'prereg-incomplete', - summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison or Diagnostics declaration is malformed', - fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, comparison, decoys, stop rules, held-out exposure) before any priced call', + summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison, Decisions, Scoring or Diagnostics declaration is malformed', + fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, decisions, arms, comparison, scoring, decoys, stop rules, held-out exposure) before any priced call: at least two decisions with different actions, and a judge from another model family checked against at least 40 labels at kappa 0.6 or better', }, // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. PREREG_DIAGNOSTIC_UNTESTED: { @@ -30,6 +30,22 @@ export const EVAL_RESULT_CODES = { summary: 'the results file has no single table with a filled row for every arm the preregistration lists, so a no-gstack row is missing or not beside its variants', fix: 'put one row per arm identity (each #no-gstack beside its variants) in one markdown table of the results file, then rerun `gstack-eval-plan results --check`; publish no number before it says ok', }, + // The result checks (docs/TESTING_INTERNALS.md#result-checks). + EVAL_JUDGE_UNCHECKED: { + anchor: 'eval-judge-unchecked', + summary: 'the preregistration names a model judge, and its hand labels are missing, fewer than the bar, all one class, or agree with the judge below the kappa bar', + fix: 'label at least the preregistered number of outputs by hand (never held-out rows), write them as {"id", "human", "judge"} JSONL, and pass them with --labels; a judge below its bar is replaced or its numbers are not published', + }, + EVAL_REPORT_UNPLANTED_ZERO: { + anchor: 'eval-report-unplanted-zero', + summary: 'a column of a results table reports a zero, and the preregistration never showed the counter behind it finding a planted case', + fix: 'declare the counter under ## Diagnostics with a synthetic fixture that plants at least one case (expected 1 or more), run prereg --check --allow-repo-commands, and name the column after the diagnostic', + }, + EVAL_REPORT_NO_ANSWER_ONLY: { + anchor: 'eval-report-no-answer-only', + summary: 'a results table counts failures without an answer-only column beside it, so a rise in failures cannot be compared with the grader\'s own reading', + fix: 'add a column headed answer-only to the same table, counting failures by the grader\'s rule on the final answer alone, and report both readings', + }, EVAL_ARM_DUPLICATE: { anchor: 'eval-arm-duplicate', summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts', diff --git a/test/eval-plan-no-gstack-arm.test.ts b/test/eval-plan-no-gstack-arm.test.ts index 1edef1d88..0d4ef9853 100644 --- a/test/eval-plan-no-gstack-arm.test.ts +++ b/test/eval-plan-no-gstack-arm.test.ts @@ -20,7 +20,8 @@ const dirs: string[] = []; const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; }; afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); -const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n'; +const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS; const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`; const LOADED = '\n## Comparison\nstack: loaded\n'; const NO_GSTACK = (o: Partial> = {}) => { @@ -189,10 +190,10 @@ describe('gstack-eval-plan prereg --check', () => { return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; }; - test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => { + test('a loaded comparison without the arm: one PREREG line out of 8, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => { const r = run(BASE + VARIANTS + LOADED); expect(r.code).toBe(1); - expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m); + expect(r.out).toMatch(/^PREREG: incomplete present=8\/8 stack=loaded no_gstack_arm=missing file=/m); expect(r.err).toContain('missing section'); expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)'); expect(r.err).not.toContain('(PREREG_INCOMPLETE)'); @@ -216,9 +217,9 @@ describe('gstack-eval-plan prereg --check', () => { test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => { const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK()); expect(ok.code, ok.out + ok.err).toBe(0); - expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m); + expect(ok.out).toMatch(/^PREREG: ok present=8\/8 stack=loaded no_gstack_arm=ok file=/m); const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n'); expect(none.code, none.out + none.err).toBe(0); - expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m); + expect(none.out).toMatch(/^PREREG: ok present=8\/8 stack=none file=/m); }); }); diff --git a/test/eval-plan-prereg-diagnostics.test.ts b/test/eval-plan-prereg-diagnostics.test.ts index 9e9e18a20..22f94abad 100644 --- a/test/eval-plan-prereg-diagnostics.test.ts +++ b/test/eval-plan-prereg-diagnostics.test.ts @@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval import { capyHost, type CapyHost } from './helpers/capy-host'; const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); -const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n'; +const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n\n## Decisions\nif the bar is met: run the full set\nif the bar is missed: fix the scorer first\n\n## Scoring\njudge: none\n'; const SUMMARIZER = [ 'touch ran.marker', 'r=$(grep -c \'"kind":"refusal"\' "$1")', @@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']])); expect(r.code).toBe(3); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file='); + expect(r.out).toContain('PREREG: ok present=8/8 stack=none diagnostics=2 file='); expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`); expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)'); expect(r.out).not.toContain('PREREG_DIAGNOSTICS:'); @@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands'); expect(r.code).toBe(1); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid'); + expect(r.out).toContain('PREREG: incomplete present=8/8 stack=none diagnostics=invalid'); expect(r.err).toContain('diagnostics: row "| refusals | two |"'); expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})'); expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows'); @@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => { test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => { const r = check(FILLED); expect(r.code).toBe(0); - expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`); + expect(r.out).toBe(`PREREG: ok present=8/8 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`); expect(parsePreregDiagnostics(FILLED)).toBeNull(); expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n\n')).toBeNull(); expect(preregTemplate()).toContain('## Diagnostics'); diff --git a/test/eval-plan-result-checks.test.ts b/test/eval-plan-result-checks.test.ts new file mode 100644 index 000000000..3b24c2a52 --- /dev/null +++ b/test/eval-plan-result-checks.test.ts @@ -0,0 +1,180 @@ +/** + * The result checks behind /eval-plan (docs/TESTING_INTERNALS.md#result-checks): + * a preregistration names the decision each result changes and its judge and + * statistics before anything is spent; a model judge's numbers count only after + * its hand labels pass; a zero comes only from a counter shown to find a planted + * case; a failures column sits beside its answer-only reading. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { checkPrereg } from '../lib/eval-plan'; +import { checkDecisions, checkJudgeLabels, checkScoring, cohensKappa, modelFamily } from '../lib/eval-plan-checks'; +import { checkResults } from '../lib/eval-plan-report'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const dirs: string[] = []; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); + +const DECISIONS = '## Decisions\nif the skill arm passes more items: keep the skill\nif there is no difference: cut the skill\n'; +const SCORING = (judge = 'gpt-6-astra', extra = 'labels: 40\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3 of 20 items\n') => `## Scoring\njudge: ${judge}\n${extra}`; +const prereg = (o: { decisions?: string; scoring?: string; frozen?: string; diagnostics?: string } = {}) => [ + o.frozen ? `frozen: ${o.frozen}\n` : '', + '## Bars\n- pass rate, higher is better\n', + o.decisions ?? DECISIONS, + '## Arms\n- claude-opus-5-5 the skill installed\n- claude-opus-5-5#plain no skill\n', + '## Comparison\nstack: none\nwhy: the scorer calibration compares prompts, not skill stacks\n', + o.scoring ?? SCORING(), + ...['Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`), + o.diagnostics ?? '', +].join('\n'); +const labels = (n: number, disagree = 0, cls = (i: number) => (i % 2 ? 'pass' : 'fail')) => + Array.from({ length: n }, (_, i) => JSON.stringify({ id: `item-${i}`, human: cls(i), judge: i < disagree ? (cls(i) === 'pass' ? 'fail' : 'pass') : cls(i) })).join('\n') + '\n'; +const table = (header: string, ...rows: string[]) => [header, header.replace(/[^|]+/g, '---'), ...rows].join('\n') + '\n'; + +describe('## Decisions', () => { + test('a passing plan names at least two results with different actions', () => { + expect(checkPrereg(prereg())).toMatchObject({ ok: true, missing: [], empty: [] }); + expect(checkDecisions(['if a: do x', 'if b: do y']).errors).toEqual([]); + }); + + test('missing, one result, an empty action, or the same action for every result fails', () => { + expect(checkPrereg(prereg({ decisions: '' })).missing).toContain('Decisions'); + expect(checkDecisions(['if it passes: ship it']).errors[0]).toContain('at least two'); + expect(checkDecisions(['if it passes: ship it', 'if it fails: tbd']).errors).toEqual(['if it fails: names no action']); + expect(checkDecisions(['if it passes: ship it', 'if it fails: Ship it']).errors[0]).toContain('changes nothing'); + expect(checkPrereg(prereg({ decisions: '## Decisions\n- we will learn a lot\n' })).ok).toBe(false); + }); +}); + +describe('## Scoring', () => { + const arms = ['claude-opus-5-5', 'claude-opus-5-5#plain']; + test('a judge from another family with 40 labels, kappa 0.6 and the paired statistics passes', () => { + expect(checkScoring(SCORING().split('\n').slice(1), arms).errors).toEqual([]); + }); + + test('a same-family judge, too few labels, a low kappa bar or missing statistics fail', () => { + expect(checkScoring(SCORING('claude-fable-5-2').split('\n').slice(1), arms).errors.join()).toContain('same model family (anthropic)'); + expect(checkScoring(SCORING('gpt-6-astra', 'labels: 39\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('at least 40'); + expect(checkScoring(SCORING('gpt-6-astra', 'labels: 40\nagreement: 0.5\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('kappa bar'); + expect(checkScoring(['judge: none'], arms).errors).toEqual(['paired: yes (every comparison is reported on paired items)', 'interval: 95% (the interval every comparison is reported with)', 'mdd: the smallest difference this run can detect, stated before the run']); + expect(checkScoring(['judge: none'], ['claude-opus-5-5']).errors).toEqual([]); + expect(checkScoring(['labels: 40'], arms).errors[0]).toContain('no judge: line'); + }); + + test('model families: labs by name, provider prefixes and loadouts dropped, else the first word', () => { + expect(['claude-opus-5-5', 'anthropic/claude-haiku-5-5', 'claude-opus-5-5#no-gstack'].map(modelFamily)).toEqual(['anthropic', 'anthropic', 'anthropic']); + expect(['gpt-6.1-sol', 'openai/gpt-6-astra', 'o3'].map(modelFamily)).toEqual(['openai', 'openai', 'openai']); + expect(['gemini-4-carbon', 'grok-4.7', 'deepseek/deepseek-v4-pro', 'particle-ai/particle-1'].map(modelFamily)).toEqual(['google', 'xai', 'deepseek', 'particle']); + }); + + test('a preregistration frozen before the result checks keeps its rules; a later freeze date does not', () => { + const old = prereg({ decisions: '', scoring: '', frozen: '2026-10-31' }); + expect(checkPrereg(old)).toMatchObject({ ok: true, grandfathered: true, frozen: '2026-10-31', missing: [] }); + expect(checkPrereg(prereg({ decisions: '', scoring: '', frozen: '2026-11-15' })).missing).toEqual(['Decisions', 'Scoring']); + }); +}); + +describe('judge labels', () => { + const scoring = { judge: 'gpt-6-astra', labels: 40, agreement: 0.6, errors: [] }; + test("Cohen's kappa: perfect, chance and undefined agreement", () => { + expect(cohensKappa([['a', 'a'], ['b', 'b']])).toBe(1); + expect(cohensKappa([['a', 'a'], ['a', 'b'], ['b', 'a'], ['b', 'b']])).toBe(0); + expect(cohensKappa([['a', 'a'], ['a', 'a']])).toBeNull(); + }); + + test('40 labels at or above the bar pass; fewer labels, a low kappa, one class or a repeated id fail', () => { + expect(checkJudgeLabels(scoring, labels(40, 4))).toMatchObject({ ok: true, n: 40, kappa: 0.8 }); + expect(checkJudgeLabels(scoring, labels(39)).problems).toEqual(['39 labelled outputs, needs 40']); + expect(checkJudgeLabels(scoring, labels(40, 10)).problems).toEqual(['kappa 0.50 is below the bar 0.6']); + expect(checkJudgeLabels(scoring, labels(40, 0, () => 'pass')).problems[0]).toContain('one class'); + expect(checkJudgeLabels(scoring, labels(40) + labels(1)).problems).toContain('id item-0 is labelled twice'); + expect(checkJudgeLabels({ ...scoring, labels: 60 }, labels(40)).problems).toEqual(['40 labelled outputs, needs 60']); + }); + + test('a row whose id, human or judge value is null, an object or empty does not count toward the bar', () => { + const bad = ['{"id": "x-1", "human": null, "judge": "pass"}', '{"id": "x-2", "human": "pass", "judge": {"v": 1}}', '{"id": {}, "human": "pass", "judge": "pass"}', '{"id": "x-4", "human": "", "judge": "pass"}']; + const r = checkJudgeLabels(scoring, labels(39) + bad.join('\n') + '\n'); + expect(r).toMatchObject({ ok: false, n: 39 }); + expect(r.problems.filter(p => p.includes('each a string, number or boolean'))).toHaveLength(4); + }); +}); + +describe('results --check: the result checks', () => { + const ROWS = table('| arm | passed | answer-only failures | failures |', '| claude-opus-5-5 | 7/9 | 2 | 3 |', '| claude-opus-5-5#plain | 5/9 | 4 | 4 |'); + test('a judged result counts only with passing labels', () => { + expect(checkResults(prereg(), ROWS).issues).toEqual([{ code: 'EVAL_JUDGE_UNCHECKED', message: 'the preregistration names judge gpt-6-astra; pass --labels ' }]); + expect(checkResults(prereg(), ROWS, labels(40, 10)).issues[0]!.code).toBe('EVAL_JUDGE_UNCHECKED'); + expect(checkResults(prereg(), ROWS, labels(40))).toMatchObject({ ok: true, issues: [] }); + expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true); + }); + + test('a zero in the arm table needs a Diagnostics counter shown to find a planted case', () => { + const zero = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 0 |'); + const diag = (n: number) => `## Diagnostics\nsummarizer: sh count.sh {fixture}\nfixture: planted.jsonl\n\n| diagnostic | expected |\n|---|---|\n| skill opens | ${n} |\n`; + expect(checkResults(prereg({ scoring: SCORING('none') }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']); + expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(0) }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']); + expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(5) }), zero).ok).toBe(true); + for (const cell of ['0%', '0/9', '0.0']) expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', `| ${cell} |`)).ok, cell).toBe(false); + expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', '| 0/6/3 |')).ok).toBe(true); + }); + + test('a zero in any table counts, not only the table that holds the arms', () => { + const arms = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 2 |'); + const other = table('| task | timeouts |', '| t1 | 0 |'); + expect(checkResults(prereg({ scoring: SCORING('none') }), arms + '\n' + other).issues).toEqual([{ code: 'EVAL_REPORT_UNPLANTED_ZERO', message: 'column "timeouts" reports a zero, and no Diagnostics counter by that name was shown to find a planted case' }]); + }); + + test('with stack: none, Decisions and Scoring errors still refuse the results', () => { + const r = checkResults(prereg({ scoring: SCORING('claude-fable-5-2') }), ROWS, labels(40)); + expect(r.ok).toBe(false); + expect(r.issues.filter(i => i.code === 'PREREG_INCOMPLETE').map(i => i.message)).toEqual(['## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5; use a judge from another family', '## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5#plain; use a judge from another family']); + expect(checkResults(prereg({ decisions: '## Decisions\n- we will learn a lot\n' }), ROWS, labels(40)).issues.map(i => i.code)).toContain('PREREG_INCOMPLETE'); + }); + + test('a failures column needs an answer-only column in the same table', () => { + const bare = table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 4 |'); + expect(checkResults(prereg({ scoring: SCORING('none') }), bare).issues).toEqual([{ code: 'EVAL_REPORT_NO_ANSWER_ONLY', message: 'column "failures" counts failures with no answer-only column beside it in the same table' }]); + expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true); + }); + + test('a grandfathered preregistration skips the result checks', () => { + const bare = table('| arm | failures |', '| claude-opus-5-5 | 0 |', '| claude-opus-5-5#plain | 4 |'); + expect(checkResults(prereg({ decisions: '', scoring: '', frozen: '2026-10-31' }), bare).ok).toBe(true); + }); +}); + +describe('gstack-eval-plan judge --check and results --labels', () => { + const run = (args: string[], files: Record) => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-result-checks-')); + dirs.push(d); + for (const [name, text] of Object.entries(files)) fs.writeFileSync(path.join(d, name), text); + const r = spawnSync(BIN, args, { cwd: d, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; + }; + + test('judge --check prints JUDGE_AGREEMENT and exits 1 with EVAL_JUDGE_UNCHECKED below the bar', () => { + const ok = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(40, 4) }); + expect(ok.code, ok.err).toBe(0); + expect(ok.out).toBe('JUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=0.80 bar=0.6\n'); + const low = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(20) }); + expect(low.code).toBe(1); + expect(low.out).toBe('JUDGE_AGREEMENT: unchecked judge=gpt-6-astra n=20/40 kappa=1.00 bar=0.6\n'); + expect(low.err).toContain('(EVAL_JUDGE_UNCHECKED)'); + expect(run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg({ scoring: SCORING('none') }), 'labels.jsonl': labels(40) }).code).toBe(2); + }); + + test('results --check takes --labels before or after the results file and prints one code per kind', () => { + const files = { 'prereg.md': prereg(), 'results.md': table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 0 |'), 'labels.jsonl': labels(40) }; + const r = run(['results', '--check', 'prereg.md', '--labels', 'labels.jsonl', 'results.md'], files); + expect(r.code).toBe(1); + expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/0 rows=0/0 file=results.md\nJUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=1.00 bar=0.6\n'); + expect(r.err).toContain('(EVAL_REPORT_UNPLANTED_ZERO)'); + expect(r.err).toContain('(EVAL_REPORT_NO_ANSWER_ONLY)'); + expect(r.err).not.toContain('(EVAL_REPORT_NO_GSTACK_ROW)'); + const noLabels = run(['results', '--check', 'prereg.md', 'results.md'], files); + expect(noLabels.err).toContain('(EVAL_JUDGE_UNCHECKED)'); + }); +}); diff --git a/test/eval-plan-results.test.ts b/test/eval-plan-results.test.ts index b4a2207a4..378645394 100644 --- a/test/eval-plan-results.test.ts +++ b/test/eval-plan-results.test.ts @@ -14,7 +14,8 @@ const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); const dirs: string[] = []; afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); -const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n'; +const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS; const PREREG = BASE + ` ## Arms - claude-opus-5-5#review-g default settings diff --git a/test/exercised-map.test.ts b/test/exercised-map.test.ts index eec13e115..8704fc1df 100644 --- a/test/exercised-map.test.ts +++ b/test/exercised-map.test.ts @@ -327,6 +327,7 @@ describe('parity with the measured prototype', () => { const REVIEWED_SUPERSET: Record = { setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there', 'lib/bun-isolation.ts': 'GSTA-56: every bun launcher (script-invocation, restamp, qa-deadline, the hooks) imports it, so it reaches every consumer of those launchers', + 'lib/eval-plan-checks.ts': 'lib/eval-plan.ts and bin/gstack-eval-plan import it (the result checks), so it reaches every consumer of the eval planner', 'lib/ship-receipt-docs.ts': 'bin/gstack-ship-receipt imports it (the docs audit record and doc caps), so it reaches every consumer of the receipt binary', 'lib/ship-receipt-prose.ts': 'bin/gstack-ship-receipt imports it (read --check-prose), so it reaches every consumer of the receipt binary', 'lib/default-base.ts': 'GST-511/GST-774: lib/ship-policy.ts (resolveBaseBranch), lib/gate-diff/git.ts, lib/pregate/context.ts, lib/ship-receipt-depth.ts and the review-plan, gate-diff, pregate, docs-candidate and diff-scope bins import or run it, so it reaches every consumer of those readers', diff --git a/test/fixtures/arm-benchmark/PREREG.md b/test/fixtures/arm-benchmark/PREREG.md index f484ac864..cc1e4987e 100644 --- a/test/fixtures/arm-benchmark/PREREG.md +++ b/test/fixtures/arm-benchmark/PREREG.md @@ -2,6 +2,8 @@ Frozen on 2026-10-31 before the first paid run of this design: three with/without pairs per task, a no-gstack arm on the pinned model, a spend ledger, and a paired judge that decides only after calibration. The instrument gates nothing; its assertions cover harness mechanics only. +frozen: 2026-10-31 + ## Bars - Metric order per arm: functional checks pass rate (tasks with an oracle), then paired over_engineering (wins/ties/losses and mean over both orders), then the per-diff armJudge mean (the no-loss diagnostic), then net LOC and tokens. - Decider bar, paired-judge calibration (`arm-pair`): before the paid run, `bun run scripts/judge-calibration.ts run --config arm-pair --budget 8` on the 16 paired items in test/fixtures/judge-calibration/arm-pair/corpus.json. The bar is `pairCalibrationBar` (test/helpers/arm-pair-judge.ts) on the `new` panels: all 6 crossing pairs decided in both orders for the less over-engineered diff, so 6/6 correct, 0 reversed, 0 tied, 0 incomplete. Position bias on the 4 identical-diff pairs is reported, not gated. diff --git a/test/fixtures/model-panel/PREREG.md b/test/fixtures/model-panel/PREREG.md index f5274e422..5d715a1c8 100644 --- a/test/fixtures/model-panel/PREREG.md +++ b/test/fixtures/model-panel/PREREG.md @@ -8,6 +8,8 @@ seen. Instrument: test/skill-e2e-model-panel.test.ts with test/helpers/model-panel-harness.ts; free selftest: test/model-panel-selftest.test.ts. +frozen: 2026-10-31 + ## Bars - Pass rate per arm: scored (non-INFRA) cells whose hidden deterministic check passes, with a Wilson 95% interval. Descriptive only: the panel gates nothing and no arm "wins". - Publication floor: an arm needs PANEL_MIN_COMPLETED = 12 scored cells before any number about it is published; below that its row says INCOMPLETE. diff --git a/test/helpers/placeholder-allowlist.ts b/test/helpers/placeholder-allowlist.ts index d9387865b..5bee10db2 100644 --- a/test/helpers/placeholder-allowlist.ts +++ b/test/helpers/placeholder-allowlist.ts @@ -149,6 +149,7 @@ export const IDENTIFIER_PLACEHOLDERS: Record = { '': { grammar: PATH, what: 'bound plan file path' }, '': { grammar: PATH, what: 'preregistration file path' }, '': { grammar: PATH, what: 'results table file path' }, + '': { grammar: PATH, what: 'hand-label JSONL file path' }, '': { grammar: PATH, what: 'sketch directory printed by an earlier block' }, '': { grammar: PATH, what: 'finalized HTML path' }, '': { grammar: PATH, what: 'saved page HTML path' }, diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index 71f78d20b..fa721ce05 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -57,7 +57,9 @@ describe('preregistration', () => { expect(check.empty.length).toBe(PREREG_SECTIONS.length); expect(check.ok).toBe(false); expect(PREREG_SECTIONS).toContain('Comparison'); - const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack'); + const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack') + .replace('## Decisions\n- filled in', '## Decisions\nif the bar is met: run the full set\nif the bar is missed: stop and fix the scorer') + .replace('## Scoring\n- filled in', '## Scoring\njudge: none'); expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] }); expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]); });