diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh
index 4a810df8a..e65205dc0 100644
--- a/bin/gstack-doctor-components.sh
+++ b/bin/gstack-doctor-components.sh
@@ -413,6 +413,9 @@ gstack_result_anchor() {
PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;;
PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;;
EVAL_REPORT_NO_GSTACK_ROW) printf '%s' 'eval-report-no-gstack-row' ;;
+ EVAL_JUDGE_UNCHECKED) printf '%s' 'eval-judge-unchecked' ;;
+ EVAL_REPORT_UNPLANTED_ZERO) printf '%s' 'eval-report-unplanted-zero' ;;
+ EVAL_REPORT_NO_ANSWER_ONLY) printf '%s' 'eval-report-no-answer-only' ;;
EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;;
OFFICE_HOURS_PROBE_REFUSED) printf '%s' 'office-hours-probe-refused' ;;
TASKS_SUPERSEDE_REFUSED) printf '%s' 'tasks-supersede-refused' ;;
diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan
index 4826b83e0..4ea39ba65 100755
--- a/bin/gstack-eval-plan
+++ b/bin/gstack-eval-plan
@@ -10,7 +10,8 @@
* gstack-eval-plan pilot settle --out
--attempt --usd
* gstack-eval-plan pilot release --out --attempt
* gstack-eval-plan pilot report --out --total [--workers W] [--json]
- * gstack-eval-plan results --check [--json]
+ * gstack-eval-plan results --check [--labels ] [--json]
+ * gstack-eval-plan judge --check --labels [--json]
* gstack-eval-plan price-gate --gate-file | --gate "" [--plan-file ] [--base ] [--cap-usd N]
* [--smoke-cap-usd N] [--allow-unpriced ""] [--out ] [--json]
*
@@ -27,13 +28,15 @@
* price-gate lint every arm's model against the base branch's
* .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete
* prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused.
- * Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts.
+ * Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts; the result checks
+ * (decisions, scoring, judge labels) in lib/eval-plan-checks.ts.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
-import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
+import { armModel, checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
import { checkResults } from '../lib/eval-plan-report';
+import { checkJudgeLabels, renderJudgeLabels } from '../lib/eval-plan-checks';
import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models';
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
import { resultLine } from '../lib/result-codes';
@@ -43,8 +46,11 @@ import { SpendCapExceeded } from '../lib/spend-ledger';
const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md)
prereg --check [--allow-repo-commands] [--json]
- the required sections exist and are filled: Bars, Arms, Comparison, Decoys, Stop rules,
- Held-out exposure, and No-gstack arm when the Comparison says stack: loaded;
+ the required sections exist and are filled: Bars, Decisions, Arms, Comparison, Scoring,
+ Decoys, Stop rules, Held-out exposure, and No-gstack arm when the Comparison says stack: loaded;
+ Decisions needs two or more "if : " lines with different actions; Scoring a
+ judge (none, or another model family with labels >= 40 and agreement >= 0.6) and, with two or
+ more arms, paired: yes, interval: 95% and mdd:; a frozen: date before 2026-11-15 skips both;
an optional Diagnostics section's summarizer runs on its synthetic fixture only with
--allow-repo-commands (else it is listed and exit 3) and every count must match
prereg --init [--title ] write the template
@@ -60,9 +66,13 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
settle unknown, and sit outside the cap and the projection until one settles a charge
pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id
pilot report --out dir --total items [--workers W] [--json] spend table + per-arm extrapolation with a 95% band, then the approval ask
- results --check [--json]
+ results --check [--labels labels.jsonl] [--json]
one table in the results file has a filled row for every Arms identity, each no-gstack arm
- beside its variants; reads text only (docs/TESTING_INTERNALS.md#no-gstack-arm)
+ beside its variants; a model judge's labels pass; a zero in any results table comes from a
+ Diagnostics counter with a planted case; a failures column has an answer-only column beside
+ it; reads text only (docs/TESTING_INTERNALS.md#result-checks)
+ judge --check --labels labels.jsonl [--json]
+ Cohen's kappa of the preregistered judge against {"id","human","judge"} hand labels
price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json]
price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base
policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
@@ -126,11 +136,11 @@ function doPrereg(rest: string[]): number {
const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null;
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n');
else {
- process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${PREREG_SECTIONS.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`);
+ process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${r.present.length + r.missing.length + r.empty.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''}${r.grandfathered ? ` frozen=${r.frozen} result_checks=exempt` : ''} file=${file}\n`);
if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`);
if (run) process.stdout.write(renderPreregDiagnostics(run));
}
- const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)];
+ const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...r.decisions.errors.map(e => `decisions: ${e}`), ...r.scoring.errors.map(e => `scoring: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)];
if (incomplete.length) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', incomplete.join('; ')) + '\n');
if (!r.no_gstack.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_NO_GSTACK_ARM', `no-gstack arm: ${r.no_gstack.problems.join('; ')}`) + '\n');
if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n');
@@ -141,12 +151,27 @@ function doPrereg(rest: string[]): number {
function doResults(rest: string[]): number {
const prereg = flag(rest, '--check');
- const results = rest.filter(a => !a.startsWith('--') && a !== prereg)[0];
+ const labels = flag(rest, '--labels');
+ const results = rest.filter(a => !a.startsWith('--') && a !== prereg && a !== labels)[0];
if (!prereg || !results || !fs.existsSync(prereg) || !fs.existsSync(results)) return usage('results needs --check ');
- const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8'));
+ if (labels !== undefined && !fs.existsSync(labels)) return usage('results --labels needs an existing labels file');
+ const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8'), labels === undefined ? null : fs.readFileSync(labels, 'utf8'));
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ prereg, results, ...r }) + '\n');
else process.stdout.write(`RESULTS: ${r.ok ? 'ok' : 'incomplete'} no_gstack_rows=${r.no_gstack.filter(id => r.found.includes(id)).length}/${r.no_gstack.length} rows=${r.found.length}/${r.required.length} file=${results}\n`);
- if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan results', 'EVAL_REPORT_NO_GSTACK_ROW', r.problems.join('; ')) + '\n'); return EXIT.fail; }
+ if (r.judge && !rest.includes('--json')) process.stdout.write(renderJudgeLabels(r.judge));
+ for (const code of new Set(r.issues.map(i => i.code))) process.stderr.write(resultLine('gstack-eval-plan results', code, r.issues.filter(i => i.code === code).map(i => i.message).join('; ')) + '\n');
+ return r.ok ? EXIT.ok : EXIT.fail;
+}
+
+function doJudge(rest: string[]): number {
+ const prereg = flag(rest, '--check');
+ const labels = flag(rest, '--labels');
+ if (!prereg || !labels || !fs.existsSync(prereg) || !fs.existsSync(labels)) return usage('judge needs --check --labels ');
+ const { scoring } = checkPrereg(fs.readFileSync(prereg, 'utf8'));
+ if (!scoring.judge || scoring.judge.toLowerCase() === 'none') return usage('judge --check needs a preregistration whose ## Scoring names a judge model');
+ const r = checkJudgeLabels(scoring, fs.readFileSync(labels, 'utf8'));
+ process.stdout.write(rest.includes('--json') ? JSON.stringify({ prereg, labels, ...r }) + '\n' : renderJudgeLabels(r));
+ if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan judge', 'EVAL_JUDGE_UNCHECKED', r.problems.join('; ')) + '\n'); return EXIT.fail; }
return EXIT.ok;
}
@@ -266,6 +291,7 @@ function main(argv: string[]): number {
if (cmd === 'pilot') return doPilot(rest);
if (cmd === 'price-gate') return doPriceGate(rest);
if (cmd === 'results') return doResults(rest);
+ if (cmd === 'judge') return doJudge(rest);
return usage(`unknown command ${cmd}`);
} catch (e: any) {
if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; }
diff --git a/changelog.d/eval-plan-result-checks.md b/changelog.d/eval-plan-result-checks.md
new file mode 100644
index 000000000..8613ccc0a
--- /dev/null
+++ b/changelog.d/eval-plan-result-checks.md
@@ -0,0 +1 @@
+- `/eval-plan` now runs four result checks instead of leaving them to judgment. `prereg --check` requires a `## Decisions` section (what each possible result changes, at least two different actions) and a `## Scoring` section (a judge from another model family than every arm, checked against at least 40 hand labels at kappa 0.6 or better, and paired results with a 95% interval and the smallest detectable difference stated before the run). The new `judge --check --labels` scores the judge against hand labels, and `results --check` holds back a judged result without passing labels, a zero from a counter never shown to find a planted case, and a failures column with no answer-only column beside it. Preregistrations frozen before 2026-11-15 keep their rules.
diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md
index c4abc8f9f..65042c1da 100644
--- a/docs/TESTING_INTERNALS.md
+++ b/docs/TESTING_INTERNALS.md
@@ -2059,6 +2059,37 @@ it as the diagnostic. Candidate folders and git identity name a project, never
the arm. The model panel (`test/skill-e2e-model-panel.test.ts`) runs a
`#no-gstack` arm beside each model (GST-468, GST-469).
+## Result checks
+
+`/eval-plan` runs four checks so that a number which cannot change a decision, or
+which comes from an unchecked instrument, never leaves the run. Each one is a
+refusal from `bin/gstack-eval-plan` (logic in `lib/eval-plan-checks.ts` and
+`lib/eval-plan-report.ts`), not advice:
+
+| Check | When it runs | Refusal |
+|---|---|---|
+| `## Decisions`: one `if : ` line per possible result, at least two, with different actions | `prereg --check`, before anything is spent | `PREREG_INCOMPLETE` |
+| `## Scoring`: `judge: none`, or a judge model from another family than every arm with `labels:` (40 or more) and `agreement:` (kappa 0.6 or more); with two or more arms, `paired: yes`, `interval: 95%` and `mdd:` stated before the run | `prereg --check` | `PREREG_INCOMPLETE` |
+| The judge's Cohen's kappa against `{"id", "human", "judge"}` hand labels, at the preregistered count and bar | `judge --check` before the approval ask; `results --check --labels` before any number goes out | `EVAL_JUDGE_UNCHECKED` |
+| A zero in any results table comes from a column named after a `## Diagnostics` counter whose synthetic fixture plants at least one case (expected 1 or more) | `results --check` | `EVAL_REPORT_UNPLANTED_ZERO` |
+| A results table with a failures column also has an `answer-only` column: failures by the grader's own rule on the final answer | `results --check` | `EVAL_REPORT_NO_ANSWER_ONLY` |
+
+Why: in our own agent teams, a counter that never fired reported zero for one
+model while an outside check with planted cases found every one, and a reported
+rise in failures vanished once failures were counted by the grader's own rule.
+A judge from the same family as the arm it grades, or one never compared with
+people, gives numbers nobody can weigh. An eval whose every result leads to the
+same action spends money to change nothing.
+
+Model families come from the model id (`claude` is Anthropic, `gpt` and `o`
+models OpenAI, `gemini` Google, `grok` xAI), else its first word. A judge
+whose labels are all one class has undefined agreement and is refused.
+
+A preregistration with a `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the
+rules it was frozen under: the PREREG line shows `result_checks=exempt` and
+none of the four checks apply. Text frozen before a priced call is not edited
+to meet a newer rule.
+
## Frontier models under test
Garry, 2026-11-01 (R54): "I think you are spending too much time with Haiku and
diff --git a/docs/skills.md b/docs/skills.md
index e45ad381c..8e2fc459f 100644
--- a/docs/skills.md
+++ b/docs/skills.md
@@ -966,7 +966,10 @@ You: /eval-plan how much will the retrieval eval cost before I run it?
`bin/gstack-eval-plan prereg --init` writes the preregistration (question, bars,
arms, decoys, stop rules, cap) and `--check` refuses to continue while a field is
-blank (`PREREG_INCOMPLETE`). `arms` resolves the `eval-arms` model role against
+blank (`PREREG_INCOMPLETE`), including the `## Decisions` and `## Scoring`
+sections the [result checks](TESTING_INTERNALS.md#result-checks) require.
+`judge --check --labels ` scores a model judge against hand
+labels (`EVAL_JUDGE_UNCHECKED`). `arms` resolves the `eval-arms` model role against
`lib/pricing.ts` and stops on an unpriced model (`EVAL_PRICE_MISSING`); `dry-run`
runs every row through the stub model for $0 and flags malformed rows
(`EVAL_DRY_RUN_FAILED`), and refuses a plan whose every arm is Haiku or Sonnet
@@ -981,7 +984,9 @@ stays out of the projection until one of its attempts settles a real charge.
A run stops at the first quota or rate-limit error on such an arm.
`results --check ` refuses a finished results file unless one
table holds a filled row for every arm, each `#no-gstack` row beside its
-variants (`EVAL_REPORT_NO_GSTACK_ROW`); no number leaves the run before it passes.
+variants (`EVAL_REPORT_NO_GSTACK_ROW`); it also checks a model judge's `--labels`,
+unplanted zeros (`EVAL_REPORT_UNPLANTED_ZERO`) and failures columns without an
+`answer-only` column (`EVAL_REPORT_NO_ANSWER_ONLY`); no number leaves the run before it passes.
The skill's closing section lists the tier-3 items (instrument audit, steady
state, sealed paths, off-machine persistence, memory preflight, power tables, QA
soak) as `tier 3, after P7`; none of them run yet.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index f14f21b3d..50c43ef6a 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -4763,12 +4763,19 @@ parentheses; grep the code.
### `PREREG_INCOMPLETE`
**Meaning.** `gstack-eval-plan prereg --check ` found a required
-preregistration section missing or empty: `## Bars`, `## Arms`,
-`## Comparison`, `## Decoys`, `## Stop rules` or `## Held-out exposure`
-(every held-out set the plan touches, its prior exposure and the reservation
-on it). It also fires when `## Comparison` has no `stack:` line, a value other
-than `loaded` or `none`, or `stack: none` without a `why:` line, and when a
-`## Diagnostics` declaration is malformed.
+preregistration section missing or empty: `## Bars`, `## Decisions`,
+`## Arms`, `## Comparison`, `## Scoring`, `## Decoys`, `## Stop rules` or
+`## Held-out exposure` (every held-out set the plan touches, its prior
+exposure and the reservation on it). It also fires when `## Comparison` has no
+`stack:` line, a value other than `loaded` or `none`, or `stack: none` without
+a `why:` line, and when a `## Diagnostics` declaration is malformed. The
+result checks add two more (`decisions:` and `scoring:` in the message):
+`## Decisions` needs two or more `if : ` lines with different
+actions, and `## Scoring` needs a `judge:` line, a model judge from another
+model family than every arm with `labels:` of 40 or more and `agreement:` of
+0.6 or more, and with two or more arms `paired: yes`, `interval: 95%` and
+`mdd:`. A `frozen:` date before 2026-11-15 exempts both
+([docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks)).
**Fix.** Fill the named sections before any priced call. A plan that cannot
name its stop rule or its held-out exposure is not preregistered; the pilot
@@ -4808,6 +4815,51 @@ in one table, and rerun the check. Publish no number before it prints
`RESULTS: ok`. A preregistration with `stack: none` needs no row. The rule is
in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm).
+
+### `EVAL_JUDGE_UNCHECKED`
+
+**Meaning.** The preregistration's `## Scoring` names a model judge, and
+`gstack-eval-plan judge --check` or `results --check` found no `--labels`
+file, fewer labelled outputs than the preregistered count (at least 40), labels
+that are all one class (agreement is undefined), a repeated id, or a Cohen's
+kappa between the hand labels and the judge below the preregistered bar (at
+least 0.6). Until it passes, the judge's numbers do not count.
+
+**Fix.** Label outputs by hand from development or calibration rows, never
+held-out ones, as JSONL rows `{"id", "human", "judge"}`, covering every kind
+of output the judge scores, and pass the file with `--labels`. A judge below
+its bar is replaced or its prompt fixed and relabelled; the bar is not
+lowered. The rule is in
+[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks).
+
+
+### `EVAL_REPORT_UNPLANTED_ZERO`
+
+**Meaning.** `gstack-eval-plan results --check` found a zero (`0`, `0%`,
+`0/N`) in a column of any results table, and the preregistration's
+`## Diagnostics` declares no counter by that column's name with a planted
+case (`expected` of 1 or more). A counter that never fires also reports zero,
+so a zero from a counter nobody saw find anything proves nothing.
+
+**Fix.** Declare the counter under `## Diagnostics` with a synthetic fixture
+that plants at least one case, run `prereg --check --allow-repo-commands`
+until its `PREREG_DIAGNOSTIC:` line says ok, and name the results column after
+the diagnostic. Otherwise leave the zero out of the report. The rule is in
+[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks).
+
+
+### `EVAL_REPORT_NO_ANSWER_ONLY`
+
+**Meaning.** A table in the results file has a column counting failures and no
+column headed `answer-only` beside it. A broad failure count (from
+transcripts, tool errors or a second rule) can rise while failures counted by
+the grader's own rule on the final answer do not move, and the two readings
+must be read side by side.
+
+**Fix.** Add an `answer-only` column to the same table, counting failures by
+the grader's rule on the final answer alone, and report both readings. The
+rule is in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#result-checks).
+
### `EVAL_ARM_DUPLICATE`
diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md
index adca9139c..9f7d7ea02 100644
--- a/eval-plan/SKILL.md
+++ b/eval-plan/SKILL.md
@@ -134,9 +134,11 @@ Write (or open) the preregistration file. Each section is required and must be f
| Section | What it holds |
|---|---|
-| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction, what a miss means for the decision |
+| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction |
+| Decisions | one `if : ` line per way the run can come out, at least two, with different actions |
| Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b |
| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
+| Scoring | `judge: none` or the judge model with `labels:` and `agreement:`; with two or more arms, `paired: yes`, `interval: 95%`, `mdd:` |
| Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm |
| Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
@@ -169,7 +171,22 @@ fixture is built to produce. The summarizer's last stdout line is one JSON objec
Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is
listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`.
`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an
-undeclared one does not match its definition. Fix that before you freeze.
+undeclared one does not match its definition. Fix that before you freeze. Any count
+the results may report as zero is declared here with a planted case: its fixture holds
+at least one, so `expected` is 1 or more and the summarizer is shown to find it.
+
+**Result checks** (docs/TESTING_INTERNALS.md#result-checks). `prereg --check` runs them;
+`PREREG_INCOMPLETE` names the line that fails:
+
+1. **Decisions before spend.** Each possible result names what we do differently. If
+ every result leads to the same action, the eval changes nothing: do not run it.
+2. **A checked judge.** A model judge comes from another model family than every arm,
+ is checked against at least 40 hand-labelled outputs (never held-out rows) at a
+ kappa of 0.6 or better, and the comparison is paired with a 95% interval and its
+ smallest detectable difference stated now.
+
+A `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the rules it was frozen under
+(`result_checks=exempt`).
### Step 1b: Arms and their price
@@ -284,6 +301,18 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
## Step 4: The approval ask
+With a model judge, check it before asking. Label at least the preregistered number of
+outputs by hand as `{"id", "human", "judge"}` JSONL (development or calibration rows),
+then:
+
+```bash
+~/.claude/skills/gstack/bin/gstack-eval-plan judge --check --labels
+```
+
+`JUDGE_AGREEMENT: ok` continues. `EVAL_JUDGE_UNCHECKED` (exit 1) means too few labels,
+one class, or kappa below the bar: replace the judge or fix its prompt, then
+relabel; never lower the bar.
+
Price every paid gate the plan names (a benchmark or eval command that spends) from the
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack). The gate
command is free text, so it reaches the pricer only as a file:
@@ -309,7 +338,8 @@ Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STAL
`--allow-unpriced ""` only on the owner's word.
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
-table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
+table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, the `JUDGE_AGREEMENT:`
+line, the preregistration's Decisions lines, and its stop rules. Then ask one
question with the projection as the recommended option only when `verdict=within`:
- A) Run `` items at the projected cost (recommended when within the cap)
@@ -328,12 +358,20 @@ from the Arms section, each `#no-gstack` row beside its variants. Before
leaves the run (a report, a READY block, a PR body), check it:
```bash
-~/.claude/skills/gstack/bin/gstack-eval-plan results --check
+~/.claude/skills/gstack/bin/gstack-eval-plan results --check [--labels ]
```
-`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm
-whose filled row is missing from the table that holds the others; add it and rerun. A
-preregistration with `stack: none` needs no row.
+`RESULTS: ok` lets the numbers go out. Each refusal exits 1:
+
+- `EVAL_REPORT_NO_GSTACK_ROW` names the arm whose filled row is missing from the table
+ that holds the others; add it. A preregistration with `stack: none` needs no row.
+- `EVAL_JUDGE_UNCHECKED`: a model judge's numbers count only with `--labels` passing.
+- `EVAL_REPORT_UNPLANTED_ZERO`: a zero in any results table, in a column not named after
+ a Diagnostics counter with a planted case. Declare and run it, or report the zero as
+ unverified, outside the tables.
+- `EVAL_REPORT_NO_ANSWER_ONLY`: a failures column needs an `answer-only` column beside
+ it (failures by the grader's own rule on the final answer); a broader count can rise
+ while that one stays flat.
The results file opens with two or three plain takeaways ("We learned X. You should do
X."): what works first, then what didn't, with its number. Cost appears only as part of a
diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl
index 4472766b0..c51321370 100644
--- a/eval-plan/SKILL.md.tmpl
+++ b/eval-plan/SKILL.md.tmpl
@@ -56,9 +56,11 @@ Write (or open) the preregistration file. Each section is required and must be f
| Section | What it holds |
|---|---|
-| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction, what a miss means for the decision |
+| Bars | `lesson:` the public lesson the run is after, then one line per metric: name, pass bar, direction |
+| Decisions | one `if : ` line per way the run can come out, at least two, with different actions |
| Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b |
| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
+| Scoring | `judge: none` or the judge model with `labels:` and `agreement:`; with two or more arms, `paired: yes`, `interval: 95%`, `mdd:` |
| Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted`, the first quota or rate-limit error on a seat-metered arm |
| Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
@@ -91,7 +93,22 @@ fixture is built to produce. The summarizer's last stdout line is one JSON objec
Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is
listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`.
`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an
-undeclared one does not match its definition. Fix that before you freeze.
+undeclared one does not match its definition. Fix that before you freeze. Any count
+the results may report as zero is declared here with a planted case: its fixture holds
+at least one, so `expected` is 1 or more and the summarizer is shown to find it.
+
+**Result checks** (docs/TESTING_INTERNALS.md#result-checks). `prereg --check` runs them;
+`PREREG_INCOMPLETE` names the line that fails:
+
+1. **Decisions before spend.** Each possible result names what we do differently. If
+ every result leads to the same action, the eval changes nothing: do not run it.
+2. **A checked judge.** A model judge comes from another model family than every arm,
+ is checked against at least 40 hand-labelled outputs (never held-out rows) at a
+ kappa of 0.6 or better, and the comparison is paired with a 95% interval and its
+ smallest detectable difference stated now.
+
+A `frozen: YYYY-MM-DD` line before 2026-11-15 keeps the rules it was frozen under
+(`result_checks=exempt`).
### Step 1b: Arms and their price
@@ -206,6 +223,18 @@ the preregistration, when `status=budget_exhausted`, or when `verdict=exceeds`.
## Step 4: The approval ask
+With a model judge, check it before asking. Label at least the preregistered number of
+outputs by hand as `{"id", "human", "judge"}` JSONL (development or calibration rows),
+then:
+
+```bash
+~/.claude/skills/gstack/bin/gstack-eval-plan judge --check --labels
+```
+
+`JUDGE_AGREEMENT: ok` continues. `EVAL_JUDGE_UNCHECKED` (exit 1) means too few labels,
+one class, or kappa below the bar: replace the judge or fix its prompt, then
+relabel; never lower the bar.
+
Price every paid gate the plan names (a benchmark or eval command that spends) from the
repo's call-plan file; nothing is executed (`docs/eval-calls.md` in gstack). The gate
command is free text, so it reaches the pricer only as a file:
@@ -224,7 +253,8 @@ Each `GATE_PRICE:` line goes into the ask. `EVAL_GATE_UNPRICED`, `EVAL_PLAN_STAL
`--allow-unpriced ""` only on the owner's word.
Present, in this order and nothing before it: the `PILOT_PROJECTION:` line, the spend
-table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, and the preregistration's stop rules. Then ask one
+table, each gate's `GATE_PRICE:` line, the pilot's per-bar results, the `JUDGE_AGREEMENT:`
+line, the preregistration's Decisions lines, and its stop rules. Then ask one
question with the projection as the recommended option only when `verdict=within`:
- A) Run `` items at the projected cost (recommended when within the cap)
@@ -243,12 +273,20 @@ from the Arms section, each `#no-gstack` row beside its variants. Before
leaves the run (a report, a READY block, a PR body), check it:
```bash
-~/.claude/skills/gstack/bin/gstack-eval-plan results --check
+~/.claude/skills/gstack/bin/gstack-eval-plan results --check [--labels ]
```
-`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm
-whose filled row is missing from the table that holds the others; add it and rerun. A
-preregistration with `stack: none` needs no row.
+`RESULTS: ok` lets the numbers go out. Each refusal exits 1:
+
+- `EVAL_REPORT_NO_GSTACK_ROW` names the arm whose filled row is missing from the table
+ that holds the others; add it. A preregistration with `stack: none` needs no row.
+- `EVAL_JUDGE_UNCHECKED`: a model judge's numbers count only with `--labels` passing.
+- `EVAL_REPORT_UNPLANTED_ZERO`: a zero in any results table, in a column not named after
+ a Diagnostics counter with a planted case. Declare and run it, or report the zero as
+ unverified, outside the tables.
+- `EVAL_REPORT_NO_ANSWER_ONLY`: a failures column needs an `answer-only` column beside
+ it (failures by the grader's own rule on the final answer); a broader count can rise
+ while that one stays flat.
The results file opens with two or three plain takeaways ("We learned X. You should do
X."): what works first, then what didn't, with its number. Cost appears only as part of a
diff --git a/lib/eval-plan-checks.ts b/lib/eval-plan-checks.ts
new file mode 100644
index 000000000..4491fc396
--- /dev/null
+++ b/lib/eval-plan-checks.ts
@@ -0,0 +1,121 @@
+/**
+ * eval-plan-checks: the result checks /eval-plan runs on top of the section
+ * check (docs/TESTING_INTERNALS.md#result-checks). `## Decisions` names what
+ * each possible result changes before anything is spent; `## Scoring` names
+ * the judge and the statistics before the run; `judge --check` scores the
+ * judge against hand labels before its numbers count. A preregistration whose
+ * `frozen:` date is before RESULT_CHECKS_SINCE keeps the rules it was frozen
+ * under. Reads text only; executes nothing.
+ */
+export const RESULT_CHECKS_SINCE = '2026-11-15';
+export const JUDGE_MIN_LABELS = 40;
+export const JUDGE_MIN_KAPPA = 0.6;
+
+const PLACEHOLDER = /^(?:|-+|—|n\/a|none|tbd|todo|\?)\.?$/i;
+const keyValues = (body: string[]) => {
+ const kv: Record = {};
+ for (const line of body) {
+ const m = /^\s*(?:[-*]\s+)?([a-z]+):(.*)$/i.exec(line);
+ if (m) kv[m[1]!.toLowerCase()] = m[2]!.trim().replace(/^`(.*)`$/, '$1');
+ }
+ return kv;
+};
+const firstNumber = (v: string | undefined) => { const m = /\d+(?:\.\d+)?/.exec(v ?? ''); return m ? Number(m[0]) : null; };
+
+/** The `frozen: YYYY-MM-DD` line, when the preregistration states its freeze date. */
+export function frozenDate(text: string): string | null {
+ return /^\s*frozen:\s*(\d{4}-\d{2}-\d{2})\s*$/im.exec(text)?.[1] ?? null;
+}
+export const grandfathered = (text: string) => { const d = frozenDate(text); return d !== null && d < RESULT_CHECKS_SINCE; };
+
+/** Model family for the judge rule: the lab behind a model id, else the id's first word. */
+export function modelFamily(model: string): string {
+ const id = model.split('#')[0]!.toLowerCase().split('/').pop()!;
+ const head = id.split(/[-_.:\s]|(?<=[a-z])(?=\d)/)[0]!;
+ const labs: Record = { claude: 'anthropic', gpt: 'openai', o: 'openai', codex: 'openai', gemini: 'google', gemma: 'google', grok: 'xai', llama: 'meta', muse: 'meta' };
+ return labs[head] ?? head;
+}
+
+export interface DecisionsCheck { outcomes: Array<{ result: string; action: string }>; errors: string[] }
+/** `## Decisions`: one `if : ` line per possible result, at least two, and not all the same action. */
+export function checkDecisions(body: string[] | null): DecisionsCheck {
+ const outcomes: DecisionsCheck['outcomes'] = [];
+ for (const line of body ?? []) {
+ const m = /^\s*(?:[-*]\s+)?if\s+(.+?):\s*(.*)$/i.exec(line);
+ if (m) outcomes.push({ result: m[1]!.trim(), action: m[2]!.trim() });
+ }
+ if (!body?.length) return { outcomes, errors: [] };
+ const errors: string[] = [];
+ if (outcomes.length < 2) errors.push('needs one "if : " line per possible result, at least two');
+ for (const o of outcomes) if (PLACEHOLDER.test(o.action)) errors.push(`if ${o.result}: names no action`);
+ const actions = new Set(outcomes.map(o => o.action.toLowerCase().replace(/\s+/g, ' ')));
+ if (outcomes.length >= 2 && actions.size === 1) errors.push('every result leads to the same action, so the eval changes nothing; do not run it');
+ return { outcomes, errors };
+}
+
+export interface ScoringCheck { judge: string | null; labels: number | null; agreement: number | null; errors: string[] }
+/**
+ * `## Scoring`: `judge: none` or the judge model; a model judge needs `labels:`
+ * (at least 40) and `agreement:` (a kappa bar of at least 0.6) and a family no
+ * arm shares. Two or more arms need `paired: yes`, `interval: 95%` and `mdd:`,
+ * the smallest difference the run can detect.
+ */
+export function checkScoring(body: string[] | null, arms: string[]): ScoringCheck {
+ if (!body?.length) return { judge: null, labels: null, agreement: null, errors: [] };
+ const kv = keyValues(body);
+ const judge = kv.judge || null;
+ const labels = firstNumber(kv.labels);
+ const agreement = firstNumber(kv.agreement);
+ const errors: string[] = [];
+ if (!judge) errors.push('no judge: line (judge: none, or the judge model)');
+ else if (judge.toLowerCase() !== 'none') {
+ if (labels === null || labels < JUDGE_MIN_LABELS) errors.push(`labels: needs at least ${JUDGE_MIN_LABELS} hand-labelled outputs the judge is checked against`);
+ if (agreement === null || agreement < JUDGE_MIN_KAPPA || agreement > 1) errors.push(`agreement: needs a kappa bar from ${JUDGE_MIN_KAPPA} to 1`);
+ const family = modelFamily(judge);
+ for (const arm of [...new Set(arms)]) if (modelFamily(arm) === family) errors.push(`judge ${judge} is the same model family (${family}) as arm ${arm}; use a judge from another family`);
+ }
+ if (new Set(arms).size >= 2) {
+ if (!/^yes\b/i.test(kv.paired ?? '')) errors.push('paired: yes (every comparison is reported on paired items)');
+ if (firstNumber(kv.interval) !== 95) errors.push('interval: 95% (the interval every comparison is reported with)');
+ if (PLACEHOLDER.test(kv.mdd ?? '')) errors.push('mdd: the smallest difference this run can detect, stated before the run');
+ }
+ return { judge, labels, agreement, errors };
+}
+
+export interface JudgeLabelCheck { ok: boolean; judge: string; n: number; kappa: number | null; need_n: number; need_kappa: number; problems: string[] }
+
+/** Cohen's kappa over paired categorical labels; null when chance agreement is total (every label one class). */
+export function cohensKappa(pairs: Array<[string, string]>): number | null {
+ if (!pairs.length) return null;
+ const n = pairs.length;
+ const a = new Map(); const b = new Map();
+ let agree = 0;
+ for (const [h, j] of pairs) { a.set(h, (a.get(h) ?? 0) + 1); b.set(j, (b.get(j) ?? 0) + 1); if (h === j) agree++; }
+ const pe = [...a.keys()].reduce((s, k) => s + (a.get(k)! / n) * ((b.get(k) ?? 0) / n), 0);
+ return pe === 1 ? null : (agree / n - pe) / (1 - pe);
+}
+
+/** Score the preregistered judge against a JSONL file of `{"id", "human", "judge"}` rows. */
+export function checkJudgeLabels(scoring: ScoringCheck, labelsText: string): JudgeLabelCheck {
+ const need_n = Math.max(JUDGE_MIN_LABELS, scoring.labels ?? 0);
+ const need_kappa = Math.max(JUDGE_MIN_KAPPA, scoring.agreement ?? 0);
+ const problems: string[] = [];
+ const pairs: Array<[string, string]> = [];
+ const ids = new Set();
+ labelsText.split('\n').forEach((line, i) => {
+ if (!line.trim()) return;
+ let row: { id?: unknown; human?: unknown; judge?: unknown } | null = null;
+ try { row = JSON.parse(line); } catch { row = null; }
+ if (!row || typeof row !== 'object' || ![row.id, row.human, row.judge].every(v => (typeof v === 'string' && v.trim() !== '') || typeof v === 'number' || typeof v === 'boolean')) { problems.push(`line ${i + 1}: needs {"id", "human", "judge"}, each a string, number or boolean`); return; }
+ if (ids.has(String(row.id))) { problems.push(`id ${String(row.id)} is labelled twice`); return; }
+ ids.add(String(row.id));
+ pairs.push([String(row.human), String(row.judge)]);
+ });
+ const kappa = cohensKappa(pairs);
+ if (pairs.length < need_n) problems.push(`${pairs.length} labelled outputs, needs ${need_n}`);
+ if (pairs.length && kappa === null) problems.push('every label is one class, so agreement is undefined; label outputs of every kind the judge scores');
+ else if (kappa !== null && kappa < need_kappa) problems.push(`kappa ${kappa.toFixed(2)} is below the bar ${need_kappa}`);
+ return { ok: problems.length === 0, judge: scoring.judge ?? 'none', n: pairs.length, kappa, need_n, need_kappa, problems };
+}
+
+export const renderJudgeLabels = (c: JudgeLabelCheck) => `JUDGE_AGREEMENT: ${c.ok ? 'ok' : 'unchecked'} judge=${c.judge} n=${c.n}/${c.need_n} kappa=${c.kappa === null ? 'undefined' : c.kappa.toFixed(2)} bar=${c.need_kappa}\n`;
diff --git a/lib/eval-plan-report.ts b/lib/eval-plan-report.ts
index fef62483f..beabde886 100644
--- a/lib/eval-plan-report.ts
+++ b/lib/eval-plan-report.ts
@@ -3,44 +3,77 @@
* A preregistration that says `stack: loaded` names its no-gstack arms; its
* results file must hold one markdown table with a filled row for every Arms
* identity, so each no-gstack row sits beside its variants
- * (docs/TESTING_INTERNALS.md#no-gstack-arm). Reads text only; executes nothing.
+ * (docs/TESTING_INTERNALS.md#no-gstack-arm). The result checks
+ * (docs/TESTING_INTERNALS.md#result-checks) hold back a model judge's numbers
+ * until its hand labels pass, a zero from a counter never shown to find a
+ * planted case, and a failures column with no answer-only reading beside it.
+ * Reads text only; executes nothing.
*/
import { armIdentities, checkPrereg } from './eval-plan';
+import { checkJudgeLabels, type JudgeLabelCheck } from './eval-plan-checks';
+import type { ResultCodeName } from './result-codes';
-export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[] }
+export interface ResultsIssue { code: ResultCodeName; message: string }
+export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[]; issues: ResultsIssue[]; judge: JudgeLabelCheck | null }
+export interface ResultTable { ids: Set; header: string[]; rows: string[][] }
const PLACEHOLDER = /^(?:|-+|—|–|n\/a|tbd|todo|\?)$/i;
+const ZERO = /^0(?:\.0+)?(?:%|\/\d+)?$/;
+const plain = (c: string) => c.replace(/[`*]/g, '').trim();
const cellsOf = (line: string) => line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(c => c.trim());
const separator = (line: string | undefined) => !!line && /^\s*\|[\s:|-]+\|?\s*$/.test(line) && line.includes('-');
-/** The filled-row identities of each markdown table: first cell, backticks stripped, header and separator rows skipped. */
-export function resultTables(text: string): Array> {
+/** Each markdown table: its header cells, data rows, and filled-row identities (first cell, backticks stripped). */
+export function resultTables(text: string): ResultTable[] {
const lines = text.split('\n');
- const tables: Array> = [];
- let current: Set | null = null;
+ const tables: ResultTable[] = [];
+ let current: ResultTable | null = null;
lines.forEach((line, i) => {
if (!/^\s*\|/.test(line)) { current = null; return; }
- if (!current) { current = new Set(); tables.push(current); }
- if (separator(line) || separator(lines[i + 1])) return;
- const [first, ...rest] = cellsOf(line);
- const id = (first ?? '').replace(/`/g, '').trim();
- if (id && rest.some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.add(id);
+ if (!current) { current = { ids: new Set(), header: [], rows: [] }; tables.push(current); }
+ if (separator(line)) return;
+ if (separator(lines[i + 1])) { current.header = cellsOf(line).map(plain); return; }
+ const cells = cellsOf(line);
+ current.rows.push(cells.map(plain));
+ const id = (cells[0] ?? '').replace(/`/g, '').trim();
+ if (id && cells.slice(1).some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.ids.add(id);
});
return tables;
}
-export function checkResults(preregText: string, resultsText: string): ResultsCheck {
+export function checkResults(preregText: string, resultsText: string, labelsText: string | null = null): ResultsCheck {
const prereg = checkPrereg(preregText);
- if (prereg.comparison.stack === 'none') return { ok: true, required: [], no_gstack: [], found: [], missing: [], problems: [] };
- const required = [...new Set(armIdentities(preregText))];
- const noGstack = prereg.no_gstack.arms;
- const problems: string[] = [];
- if (!prereg.ok) problems.push('the preregistration does not pass prereg --check');
+ const issues: ResultsIssue[] = [];
const tables = resultTables(resultsText);
- const best = tables.reduce>((b, t) => (required.filter(id => t.has(id)).length > required.filter(id => b.has(id)).length ? t : b), new Set());
- const found = required.filter(id => best.has(id));
- const missing = required.filter(id => !best.has(id));
- if (!tables.length) problems.push('no markdown table in the results file');
- else for (const id of missing) problems.push(`no filled row for ${id} in the table that holds the other arms`);
- return { ok: problems.length === 0, required, no_gstack: noGstack, found, missing, problems };
+ const arms = [...new Set(armIdentities(preregText))];
+ const required = prereg.comparison.stack === 'none' ? [] : arms;
+ const best = tables.reduce((b, t) => (!b || arms.filter(id => t.ids.has(id)).length > arms.filter(id => b.ids.has(id)).length ? t : b), null);
+ const found = required.filter(id => best?.ids.has(id));
+ const missing = required.filter(id => !best?.ids.has(id));
+ if (prereg.comparison.stack !== 'none') {
+ if (!prereg.ok) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: 'the preregistration does not pass prereg --check' });
+ if (!tables.length) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: 'no markdown table in the results file' });
+ else for (const id of missing) issues.push({ code: 'EVAL_REPORT_NO_GSTACK_ROW', message: `no filled row for ${id} in the table that holds the other arms` });
+ }
+ let judge: JudgeLabelCheck | null = null;
+ if (!prereg.grandfathered) {
+ for (const [section, errors] of [['Decisions', prereg.decisions.errors], ['Scoring', prereg.scoring.errors]] as const) {
+ for (const e of errors) issues.push({ code: 'PREREG_INCOMPLETE', message: `## ${section}: ${e}` });
+ }
+ if (prereg.scoring.judge && prereg.scoring.judge.toLowerCase() !== 'none') {
+ judge = labelsText === null ? null : checkJudgeLabels(prereg.scoring, labelsText);
+ if (!judge) issues.push({ code: 'EVAL_JUDGE_UNCHECKED', message: `the preregistration names judge ${prereg.scoring.judge}; pass --labels ` });
+ else if (!judge.ok) issues.push({ code: 'EVAL_JUDGE_UNCHECKED', message: judge.problems.join('; ') });
+ }
+ const planted = new Set((prereg.diagnostics?.expected ?? []).filter(e => e.count >= 1).map(e => e.name.toLowerCase()));
+ for (const t of tables) {
+ t.header.forEach((name, col) => {
+ if (col === 0 || planted.has(name.toLowerCase()) || !t.rows.some(r => ZERO.test(r[col] ?? ''))) return;
+ issues.push({ code: 'EVAL_REPORT_UNPLANTED_ZERO', message: `column "${name}" reports a zero, and no Diagnostics counter by that name was shown to find a planted case` });
+ });
+ const failures = t.header.filter(h => /\bfail/i.test(h) && !/answer[- ]only/i.test(h));
+ if (failures.length && !t.header.some(h => /answer[- ]only/i.test(h))) issues.push({ code: 'EVAL_REPORT_NO_ANSWER_ONLY', message: `column "${failures[0]}" counts failures with no answer-only column beside it in the same table` });
+ }
+ }
+ return { ok: issues.length === 0, required, no_gstack: prereg.no_gstack.arms, found, missing, problems: issues.map(i => i.message), issues, judge };
}
diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts
index 258ee975a..9b5580c1e 100644
--- a/lib/eval-plan.ts
+++ b/lib/eval-plan.ts
@@ -19,16 +19,19 @@ import { EXCLUDED_BILLING_CATEGORIES, estimateCostUsd, hasPricing, PRICING, pric
import type { ResultCodeName } from './result-codes';
import type { ModelRulesCheck } from './eval-models';
import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type SpendLedger, type SpendSummary } from './spend-ledger';
+import { checkDecisions, checkScoring, frozenDate, grandfathered, type DecisionsCheck, type ScoringCheck } from './eval-plan-checks';
// ---------------------------------------------------------------------------
// preregistration
// ---------------------------------------------------------------------------
-export const PREREG_SECTIONS = ['Bars', 'Arms', 'Comparison', 'Decoys', 'Stop rules', 'Held-out exposure'] as const;
+export const PREREG_SECTIONS = ['Bars', 'Decisions', 'Arms', 'Comparison', 'Scoring', 'Decoys', 'Stop rules', 'Held-out exposure'] as const;
+/** Sections added with the result checks; a preregistration frozen before them (`frozen:` before RESULT_CHECKS_SINCE) is not asked for them. */
+const RESULT_CHECK_SECTIONS: readonly string[] = ['Decisions', 'Scoring'];
/** `## Comparison`: `stack: loaded` when any arm loads gstack or another skill stack, `stack: none` with a `why:` otherwise. */
export interface ComparisonCheck { stack: 'loaded' | 'none' | null; why: string | null; errors: string[] }
/** `## No-gstack arm`, required when `stack: loaded`: one `#no-gstack` Arms line per model, the inputs, prompt, panel and report it shares with the variants, and its masking, install and audit lines. */
export interface NoGstackArmCheck { required: boolean; ok: boolean; arms: string[]; problems: string[] }
-export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null }
+export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null; decisions: DecisionsCheck; scoring: ScoringCheck; frozen: string | null; grandfathered: boolean }
export const NO_GSTACK_LOADOUT = 'no-gstack';
/** masking, install and audit keep the arm clean (TESTING_INTERNALS.md#no-gstack-arm): an arm that read a gstack skill file is contaminated. */
export const NO_GSTACK_KEYS = ['arms', 'inputs', 'prompt', 'panel', 'report', 'masking', 'install', 'audit'] as const;
@@ -37,7 +40,10 @@ export function preregTemplate(title = 'Eval preregistration'): string {
return `# ${title}
## Bars
-
+
+
+## Decisions
+
## Arms
@@ -48,6 +54,9 @@ export function preregTemplate(title = 'Eval preregistration'): string {
## No-gstack arm
+## Scoring
+
+
## Decoys
@@ -56,7 +65,7 @@ export function preregTemplate(title = 'Eval preregistration'): string {
## Held-out exposure
-
+
`;
}
@@ -130,14 +139,19 @@ function checkNoGstackArm(text: string, stack: ComparisonCheck['stack']): NoGsta
export function checkPrereg(text: string): PreregCheck {
const present: string[] = []; const missing: string[] = []; const empty: string[] = [];
+ const exempt = grandfathered(text);
for (const name of PREREG_SECTIONS) {
+ if (exempt && RESULT_CHECK_SECTIONS.includes(name)) continue;
const body = sectionBody(text, name);
if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name);
}
const comparison = checkComparison(text);
const no_gstack = checkNoGstackArm(text, comparison.stack);
const diagnostics = parsePreregDiagnostics(text);
- return { ok: missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length, present, missing, empty, comparison, no_gstack, diagnostics };
+ const decisions = exempt ? { outcomes: [], errors: [] } : checkDecisions(sectionBody(text, 'Decisions'));
+ const scoring = exempt ? { judge: null, labels: null, agreement: null, errors: [] } : checkScoring(sectionBody(text, 'Scoring'), armIdentities(text));
+ const ok = missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length && !decisions.errors.length && !scoring.errors.length;
+ return { ok, present, missing, empty, comparison, no_gstack, diagnostics, decisions, scoring, frozen: frozenDate(text), grandfathered: exempt };
}
// ---------------------------------------------------------------------------
diff --git a/lib/result-codes-eval.ts b/lib/result-codes-eval.ts
index e019e3bdd..23d6143cf 100644
--- a/lib/result-codes-eval.ts
+++ b/lib/result-codes-eval.ts
@@ -10,8 +10,8 @@ import type { ResultCode } from './result-codes';
export const EVAL_RESULT_CODES = {
PREREG_INCOMPLETE: {
anchor: 'prereg-incomplete',
- summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison or Diagnostics declaration is malformed',
- fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, comparison, decoys, stop rules, held-out exposure) before any priced call',
+ summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison, Decisions, Scoring or Diagnostics declaration is malformed',
+ fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, decisions, arms, comparison, scoring, decoys, stop rules, held-out exposure) before any priced call: at least two decisions with different actions, and a judge from another model family checked against at least 40 labels at kappa 0.6 or better',
},
// Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit.
PREREG_DIAGNOSTIC_UNTESTED: {
@@ -30,6 +30,22 @@ export const EVAL_RESULT_CODES = {
summary: 'the results file has no single table with a filled row for every arm the preregistration lists, so a no-gstack row is missing or not beside its variants',
fix: 'put one row per arm identity (each #no-gstack beside its variants) in one markdown table of the results file, then rerun `gstack-eval-plan results --check`; publish no number before it says ok',
},
+ // The result checks (docs/TESTING_INTERNALS.md#result-checks).
+ EVAL_JUDGE_UNCHECKED: {
+ anchor: 'eval-judge-unchecked',
+ summary: 'the preregistration names a model judge, and its hand labels are missing, fewer than the bar, all one class, or agree with the judge below the kappa bar',
+ fix: 'label at least the preregistered number of outputs by hand (never held-out rows), write them as {"id", "human", "judge"} JSONL, and pass them with --labels; a judge below its bar is replaced or its numbers are not published',
+ },
+ EVAL_REPORT_UNPLANTED_ZERO: {
+ anchor: 'eval-report-unplanted-zero',
+ summary: 'a column of a results table reports a zero, and the preregistration never showed the counter behind it finding a planted case',
+ fix: 'declare the counter under ## Diagnostics with a synthetic fixture that plants at least one case (expected 1 or more), run prereg --check --allow-repo-commands, and name the column after the diagnostic',
+ },
+ EVAL_REPORT_NO_ANSWER_ONLY: {
+ anchor: 'eval-report-no-answer-only',
+ summary: 'a results table counts failures without an answer-only column beside it, so a rise in failures cannot be compared with the grader\'s own reading',
+ fix: 'add a column headed answer-only to the same table, counting failures by the grader\'s rule on the final answer alone, and report both readings',
+ },
EVAL_ARM_DUPLICATE: {
anchor: 'eval-arm-duplicate',
summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts',
diff --git a/test/eval-plan-no-gstack-arm.test.ts b/test/eval-plan-no-gstack-arm.test.ts
index 1edef1d88..0d4ef9853 100644
--- a/test/eval-plan-no-gstack-arm.test.ts
+++ b/test/eval-plan-no-gstack-arm.test.ts
@@ -20,7 +20,8 @@ const dirs: string[] = [];
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; };
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
-const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
+const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n';
+const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS;
const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`;
const LOADED = '\n## Comparison\nstack: loaded\n';
const NO_GSTACK = (o: Partial> = {}) => {
@@ -189,10 +190,10 @@ describe('gstack-eval-plan prereg --check', () => {
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
};
- test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => {
+ test('a loaded comparison without the arm: one PREREG line out of 8, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => {
const r = run(BASE + VARIANTS + LOADED);
expect(r.code).toBe(1);
- expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m);
+ expect(r.out).toMatch(/^PREREG: incomplete present=8\/8 stack=loaded no_gstack_arm=missing file=/m);
expect(r.err).toContain('missing section');
expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)');
expect(r.err).not.toContain('(PREREG_INCOMPLETE)');
@@ -216,9 +217,9 @@ describe('gstack-eval-plan prereg --check', () => {
test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => {
const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK());
expect(ok.code, ok.out + ok.err).toBe(0);
- expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m);
+ expect(ok.out).toMatch(/^PREREG: ok present=8\/8 stack=loaded no_gstack_arm=ok file=/m);
const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n');
expect(none.code, none.out + none.err).toBe(0);
- expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m);
+ expect(none.out).toMatch(/^PREREG: ok present=8\/8 stack=none file=/m);
});
});
diff --git a/test/eval-plan-prereg-diagnostics.test.ts b/test/eval-plan-prereg-diagnostics.test.ts
index 9e9e18a20..22f94abad 100644
--- a/test/eval-plan-prereg-diagnostics.test.ts
+++ b/test/eval-plan-prereg-diagnostics.test.ts
@@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval
import { capyHost, type CapyHost } from './helpers/capy-host';
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
-const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n';
+const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n\n## Decisions\nif the bar is met: run the full set\nif the bar is missed: fix the scorer first\n\n## Scoring\njudge: none\n';
const SUMMARIZER = [
'touch ran.marker',
'r=$(grep -c \'"kind":"refusal"\' "$1")',
@@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => {
const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]));
expect(r.code).toBe(3);
expect(r.ran).toBe(false);
- expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file=');
+ expect(r.out).toContain('PREREG: ok present=8/8 stack=none diagnostics=2 file=');
expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`);
expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)');
expect(r.out).not.toContain('PREREG_DIAGNOSTICS:');
@@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => {
const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands');
expect(r.code).toBe(1);
expect(r.ran).toBe(false);
- expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid');
+ expect(r.out).toContain('PREREG: incomplete present=8/8 stack=none diagnostics=invalid');
expect(r.err).toContain('diagnostics: row "| refusals | two |"');
expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})');
expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows');
@@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => {
test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => {
const r = check(FILLED);
expect(r.code).toBe(0);
- expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`);
+ expect(r.out).toBe(`PREREG: ok present=8/8 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`);
expect(parsePreregDiagnostics(FILLED)).toBeNull();
expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n\n')).toBeNull();
expect(preregTemplate()).toContain('## Diagnostics');
diff --git a/test/eval-plan-result-checks.test.ts b/test/eval-plan-result-checks.test.ts
new file mode 100644
index 000000000..3b24c2a52
--- /dev/null
+++ b/test/eval-plan-result-checks.test.ts
@@ -0,0 +1,180 @@
+/**
+ * The result checks behind /eval-plan (docs/TESTING_INTERNALS.md#result-checks):
+ * a preregistration names the decision each result changes and its judge and
+ * statistics before anything is spent; a model judge's numbers count only after
+ * its hand labels pass; a zero comes only from a counter shown to find a planted
+ * case; a failures column sits beside its answer-only reading.
+ */
+import { afterEach, describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { checkPrereg } from '../lib/eval-plan';
+import { checkDecisions, checkJudgeLabels, checkScoring, cohensKappa, modelFamily } from '../lib/eval-plan-checks';
+import { checkResults } from '../lib/eval-plan-report';
+
+const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
+const dirs: string[] = [];
+afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
+
+const DECISIONS = '## Decisions\nif the skill arm passes more items: keep the skill\nif there is no difference: cut the skill\n';
+const SCORING = (judge = 'gpt-6-astra', extra = 'labels: 40\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3 of 20 items\n') => `## Scoring\njudge: ${judge}\n${extra}`;
+const prereg = (o: { decisions?: string; scoring?: string; frozen?: string; diagnostics?: string } = {}) => [
+ o.frozen ? `frozen: ${o.frozen}\n` : '',
+ '## Bars\n- pass rate, higher is better\n',
+ o.decisions ?? DECISIONS,
+ '## Arms\n- claude-opus-5-5 the skill installed\n- claude-opus-5-5#plain no skill\n',
+ '## Comparison\nstack: none\nwhy: the scorer calibration compares prompts, not skill stacks\n',
+ o.scoring ?? SCORING(),
+ ...['Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`),
+ o.diagnostics ?? '',
+].join('\n');
+const labels = (n: number, disagree = 0, cls = (i: number) => (i % 2 ? 'pass' : 'fail')) =>
+ Array.from({ length: n }, (_, i) => JSON.stringify({ id: `item-${i}`, human: cls(i), judge: i < disagree ? (cls(i) === 'pass' ? 'fail' : 'pass') : cls(i) })).join('\n') + '\n';
+const table = (header: string, ...rows: string[]) => [header, header.replace(/[^|]+/g, '---'), ...rows].join('\n') + '\n';
+
+describe('## Decisions', () => {
+ test('a passing plan names at least two results with different actions', () => {
+ expect(checkPrereg(prereg())).toMatchObject({ ok: true, missing: [], empty: [] });
+ expect(checkDecisions(['if a: do x', 'if b: do y']).errors).toEqual([]);
+ });
+
+ test('missing, one result, an empty action, or the same action for every result fails', () => {
+ expect(checkPrereg(prereg({ decisions: '' })).missing).toContain('Decisions');
+ expect(checkDecisions(['if it passes: ship it']).errors[0]).toContain('at least two');
+ expect(checkDecisions(['if it passes: ship it', 'if it fails: tbd']).errors).toEqual(['if it fails: names no action']);
+ expect(checkDecisions(['if it passes: ship it', 'if it fails: Ship it']).errors[0]).toContain('changes nothing');
+ expect(checkPrereg(prereg({ decisions: '## Decisions\n- we will learn a lot\n' })).ok).toBe(false);
+ });
+});
+
+describe('## Scoring', () => {
+ const arms = ['claude-opus-5-5', 'claude-opus-5-5#plain'];
+ test('a judge from another family with 40 labels, kappa 0.6 and the paired statistics passes', () => {
+ expect(checkScoring(SCORING().split('\n').slice(1), arms).errors).toEqual([]);
+ });
+
+ test('a same-family judge, too few labels, a low kappa bar or missing statistics fail', () => {
+ expect(checkScoring(SCORING('claude-fable-5-2').split('\n').slice(1), arms).errors.join()).toContain('same model family (anthropic)');
+ expect(checkScoring(SCORING('gpt-6-astra', 'labels: 39\nagreement: 0.6\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('at least 40');
+ expect(checkScoring(SCORING('gpt-6-astra', 'labels: 40\nagreement: 0.5\npaired: yes\ninterval: 95%\nmdd: 3\n').split('\n').slice(1), arms).errors[0]).toContain('kappa bar');
+ expect(checkScoring(['judge: none'], arms).errors).toEqual(['paired: yes (every comparison is reported on paired items)', 'interval: 95% (the interval every comparison is reported with)', 'mdd: the smallest difference this run can detect, stated before the run']);
+ expect(checkScoring(['judge: none'], ['claude-opus-5-5']).errors).toEqual([]);
+ expect(checkScoring(['labels: 40'], arms).errors[0]).toContain('no judge: line');
+ });
+
+ test('model families: labs by name, provider prefixes and loadouts dropped, else the first word', () => {
+ expect(['claude-opus-5-5', 'anthropic/claude-haiku-5-5', 'claude-opus-5-5#no-gstack'].map(modelFamily)).toEqual(['anthropic', 'anthropic', 'anthropic']);
+ expect(['gpt-6.1-sol', 'openai/gpt-6-astra', 'o3'].map(modelFamily)).toEqual(['openai', 'openai', 'openai']);
+ expect(['gemini-4-carbon', 'grok-4.7', 'deepseek/deepseek-v4-pro', 'particle-ai/particle-1'].map(modelFamily)).toEqual(['google', 'xai', 'deepseek', 'particle']);
+ });
+
+ test('a preregistration frozen before the result checks keeps its rules; a later freeze date does not', () => {
+ const old = prereg({ decisions: '', scoring: '', frozen: '2026-10-31' });
+ expect(checkPrereg(old)).toMatchObject({ ok: true, grandfathered: true, frozen: '2026-10-31', missing: [] });
+ expect(checkPrereg(prereg({ decisions: '', scoring: '', frozen: '2026-11-15' })).missing).toEqual(['Decisions', 'Scoring']);
+ });
+});
+
+describe('judge labels', () => {
+ const scoring = { judge: 'gpt-6-astra', labels: 40, agreement: 0.6, errors: [] };
+ test("Cohen's kappa: perfect, chance and undefined agreement", () => {
+ expect(cohensKappa([['a', 'a'], ['b', 'b']])).toBe(1);
+ expect(cohensKappa([['a', 'a'], ['a', 'b'], ['b', 'a'], ['b', 'b']])).toBe(0);
+ expect(cohensKappa([['a', 'a'], ['a', 'a']])).toBeNull();
+ });
+
+ test('40 labels at or above the bar pass; fewer labels, a low kappa, one class or a repeated id fail', () => {
+ expect(checkJudgeLabels(scoring, labels(40, 4))).toMatchObject({ ok: true, n: 40, kappa: 0.8 });
+ expect(checkJudgeLabels(scoring, labels(39)).problems).toEqual(['39 labelled outputs, needs 40']);
+ expect(checkJudgeLabels(scoring, labels(40, 10)).problems).toEqual(['kappa 0.50 is below the bar 0.6']);
+ expect(checkJudgeLabels(scoring, labels(40, 0, () => 'pass')).problems[0]).toContain('one class');
+ expect(checkJudgeLabels(scoring, labels(40) + labels(1)).problems).toContain('id item-0 is labelled twice');
+ expect(checkJudgeLabels({ ...scoring, labels: 60 }, labels(40)).problems).toEqual(['40 labelled outputs, needs 60']);
+ });
+
+ test('a row whose id, human or judge value is null, an object or empty does not count toward the bar', () => {
+ const bad = ['{"id": "x-1", "human": null, "judge": "pass"}', '{"id": "x-2", "human": "pass", "judge": {"v": 1}}', '{"id": {}, "human": "pass", "judge": "pass"}', '{"id": "x-4", "human": "", "judge": "pass"}'];
+ const r = checkJudgeLabels(scoring, labels(39) + bad.join('\n') + '\n');
+ expect(r).toMatchObject({ ok: false, n: 39 });
+ expect(r.problems.filter(p => p.includes('each a string, number or boolean'))).toHaveLength(4);
+ });
+});
+
+describe('results --check: the result checks', () => {
+ const ROWS = table('| arm | passed | answer-only failures | failures |', '| claude-opus-5-5 | 7/9 | 2 | 3 |', '| claude-opus-5-5#plain | 5/9 | 4 | 4 |');
+ test('a judged result counts only with passing labels', () => {
+ expect(checkResults(prereg(), ROWS).issues).toEqual([{ code: 'EVAL_JUDGE_UNCHECKED', message: 'the preregistration names judge gpt-6-astra; pass --labels ' }]);
+ expect(checkResults(prereg(), ROWS, labels(40, 10)).issues[0]!.code).toBe('EVAL_JUDGE_UNCHECKED');
+ expect(checkResults(prereg(), ROWS, labels(40))).toMatchObject({ ok: true, issues: [] });
+ expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true);
+ });
+
+ test('a zero in the arm table needs a Diagnostics counter shown to find a planted case', () => {
+ const zero = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 0 |');
+ const diag = (n: number) => `## Diagnostics\nsummarizer: sh count.sh {fixture}\nfixture: planted.jsonl\n\n| diagnostic | expected |\n|---|---|\n| skill opens | ${n} |\n`;
+ expect(checkResults(prereg({ scoring: SCORING('none') }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']);
+ expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(0) }), zero).issues.map(i => i.code)).toEqual(['EVAL_REPORT_UNPLANTED_ZERO']);
+ expect(checkResults(prereg({ scoring: SCORING('none'), diagnostics: diag(5) }), zero).ok).toBe(true);
+ for (const cell of ['0%', '0/9', '0.0']) expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', `| ${cell} |`)).ok, cell).toBe(false);
+ expect(checkResults(prereg({ scoring: SCORING('none') }), zero.replace('| 0 |', '| 0/6/3 |')).ok).toBe(true);
+ });
+
+ test('a zero in any table counts, not only the table that holds the arms', () => {
+ const arms = table('| arm | skill opens |', '| claude-opus-5-5 | 4 |', '| claude-opus-5-5#plain | 2 |');
+ const other = table('| task | timeouts |', '| t1 | 0 |');
+ expect(checkResults(prereg({ scoring: SCORING('none') }), arms + '\n' + other).issues).toEqual([{ code: 'EVAL_REPORT_UNPLANTED_ZERO', message: 'column "timeouts" reports a zero, and no Diagnostics counter by that name was shown to find a planted case' }]);
+ });
+
+ test('with stack: none, Decisions and Scoring errors still refuse the results', () => {
+ const r = checkResults(prereg({ scoring: SCORING('claude-fable-5-2') }), ROWS, labels(40));
+ expect(r.ok).toBe(false);
+ expect(r.issues.filter(i => i.code === 'PREREG_INCOMPLETE').map(i => i.message)).toEqual(['## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5; use a judge from another family', '## Scoring: judge claude-fable-5-2 is the same model family (anthropic) as arm claude-opus-5-5#plain; use a judge from another family']);
+ expect(checkResults(prereg({ decisions: '## Decisions\n- we will learn a lot\n' }), ROWS, labels(40)).issues.map(i => i.code)).toContain('PREREG_INCOMPLETE');
+ });
+
+ test('a failures column needs an answer-only column in the same table', () => {
+ const bare = table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 4 |');
+ expect(checkResults(prereg({ scoring: SCORING('none') }), bare).issues).toEqual([{ code: 'EVAL_REPORT_NO_ANSWER_ONLY', message: 'column "failures" counts failures with no answer-only column beside it in the same table' }]);
+ expect(checkResults(prereg({ scoring: SCORING('none') }), ROWS).ok).toBe(true);
+ });
+
+ test('a grandfathered preregistration skips the result checks', () => {
+ const bare = table('| arm | failures |', '| claude-opus-5-5 | 0 |', '| claude-opus-5-5#plain | 4 |');
+ expect(checkResults(prereg({ decisions: '', scoring: '', frozen: '2026-10-31' }), bare).ok).toBe(true);
+ });
+});
+
+describe('gstack-eval-plan judge --check and results --labels', () => {
+ const run = (args: string[], files: Record) => {
+ const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-result-checks-'));
+ dirs.push(d);
+ for (const [name, text] of Object.entries(files)) fs.writeFileSync(path.join(d, name), text);
+ const r = spawnSync(BIN, args, { cwd: d, encoding: 'utf8', timeout: 60_000 });
+ return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
+ };
+
+ test('judge --check prints JUDGE_AGREEMENT and exits 1 with EVAL_JUDGE_UNCHECKED below the bar', () => {
+ const ok = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(40, 4) });
+ expect(ok.code, ok.err).toBe(0);
+ expect(ok.out).toBe('JUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=0.80 bar=0.6\n');
+ const low = run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg(), 'labels.jsonl': labels(20) });
+ expect(low.code).toBe(1);
+ expect(low.out).toBe('JUDGE_AGREEMENT: unchecked judge=gpt-6-astra n=20/40 kappa=1.00 bar=0.6\n');
+ expect(low.err).toContain('(EVAL_JUDGE_UNCHECKED)');
+ expect(run(['judge', '--check', 'prereg.md', '--labels', 'labels.jsonl'], { 'prereg.md': prereg({ scoring: SCORING('none') }), 'labels.jsonl': labels(40) }).code).toBe(2);
+ });
+
+ test('results --check takes --labels before or after the results file and prints one code per kind', () => {
+ const files = { 'prereg.md': prereg(), 'results.md': table('| arm | failures |', '| claude-opus-5-5 | 3 |', '| claude-opus-5-5#plain | 0 |'), 'labels.jsonl': labels(40) };
+ const r = run(['results', '--check', 'prereg.md', '--labels', 'labels.jsonl', 'results.md'], files);
+ expect(r.code).toBe(1);
+ expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/0 rows=0/0 file=results.md\nJUDGE_AGREEMENT: ok judge=gpt-6-astra n=40/40 kappa=1.00 bar=0.6\n');
+ expect(r.err).toContain('(EVAL_REPORT_UNPLANTED_ZERO)');
+ expect(r.err).toContain('(EVAL_REPORT_NO_ANSWER_ONLY)');
+ expect(r.err).not.toContain('(EVAL_REPORT_NO_GSTACK_ROW)');
+ const noLabels = run(['results', '--check', 'prereg.md', 'results.md'], files);
+ expect(noLabels.err).toContain('(EVAL_JUDGE_UNCHECKED)');
+ });
+});
diff --git a/test/eval-plan-results.test.ts b/test/eval-plan-results.test.ts
index b4a2207a4..378645394 100644
--- a/test/eval-plan-results.test.ts
+++ b/test/eval-plan-results.test.ts
@@ -14,7 +14,8 @@ const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
const dirs: string[] = [];
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
-const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
+const RESULT_SECTIONS = '## Decisions\nif the variant passes more items: adopt it\nif there is no difference: keep the default\n\n## Scoring\njudge: none\npaired: yes\ninterval: 95%\nmdd: 2 of 9 items\n';
+const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n' + RESULT_SECTIONS;
const PREREG = BASE + `
## Arms
- claude-opus-5-5#review-g default settings
diff --git a/test/exercised-map.test.ts b/test/exercised-map.test.ts
index eec13e115..8704fc1df 100644
--- a/test/exercised-map.test.ts
+++ b/test/exercised-map.test.ts
@@ -327,6 +327,7 @@ describe('parity with the measured prototype', () => {
const REVIEWED_SUPERSET: Record = {
setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there',
'lib/bun-isolation.ts': 'GSTA-56: every bun launcher (script-invocation, restamp, qa-deadline, the hooks) imports it, so it reaches every consumer of those launchers',
+ 'lib/eval-plan-checks.ts': 'lib/eval-plan.ts and bin/gstack-eval-plan import it (the result checks), so it reaches every consumer of the eval planner',
'lib/ship-receipt-docs.ts': 'bin/gstack-ship-receipt imports it (the docs audit record and doc caps), so it reaches every consumer of the receipt binary',
'lib/ship-receipt-prose.ts': 'bin/gstack-ship-receipt imports it (read --check-prose), so it reaches every consumer of the receipt binary',
'lib/default-base.ts': 'GST-511/GST-774: lib/ship-policy.ts (resolveBaseBranch), lib/gate-diff/git.ts, lib/pregate/context.ts, lib/ship-receipt-depth.ts and the review-plan, gate-diff, pregate, docs-candidate and diff-scope bins import or run it, so it reaches every consumer of those readers',
diff --git a/test/fixtures/arm-benchmark/PREREG.md b/test/fixtures/arm-benchmark/PREREG.md
index f484ac864..cc1e4987e 100644
--- a/test/fixtures/arm-benchmark/PREREG.md
+++ b/test/fixtures/arm-benchmark/PREREG.md
@@ -2,6 +2,8 @@
Frozen on 2026-10-31 before the first paid run of this design: three with/without pairs per task, a no-gstack arm on the pinned model, a spend ledger, and a paired judge that decides only after calibration. The instrument gates nothing; its assertions cover harness mechanics only.
+frozen: 2026-10-31
+
## Bars
- Metric order per arm: functional checks pass rate (tasks with an oracle), then paired over_engineering (wins/ties/losses and mean over both orders), then the per-diff armJudge mean (the no-loss diagnostic), then net LOC and tokens.
- Decider bar, paired-judge calibration (`arm-pair`): before the paid run, `bun run scripts/judge-calibration.ts run --config arm-pair --budget 8` on the 16 paired items in test/fixtures/judge-calibration/arm-pair/corpus.json. The bar is `pairCalibrationBar` (test/helpers/arm-pair-judge.ts) on the `new` panels: all 6 crossing pairs decided in both orders for the less over-engineered diff, so 6/6 correct, 0 reversed, 0 tied, 0 incomplete. Position bias on the 4 identical-diff pairs is reported, not gated.
diff --git a/test/fixtures/model-panel/PREREG.md b/test/fixtures/model-panel/PREREG.md
index f5274e422..5d715a1c8 100644
--- a/test/fixtures/model-panel/PREREG.md
+++ b/test/fixtures/model-panel/PREREG.md
@@ -8,6 +8,8 @@ seen. Instrument: test/skill-e2e-model-panel.test.ts with
test/helpers/model-panel-harness.ts; free selftest:
test/model-panel-selftest.test.ts.
+frozen: 2026-10-31
+
## Bars
- Pass rate per arm: scored (non-INFRA) cells whose hidden deterministic check passes, with a Wilson 95% interval. Descriptive only: the panel gates nothing and no arm "wins".
- Publication floor: an arm needs PANEL_MIN_COMPLETED = 12 scored cells before any number about it is published; below that its row says INCOMPLETE.
diff --git a/test/helpers/placeholder-allowlist.ts b/test/helpers/placeholder-allowlist.ts
index d9387865b..5bee10db2 100644
--- a/test/helpers/placeholder-allowlist.ts
+++ b/test/helpers/placeholder-allowlist.ts
@@ -149,6 +149,7 @@ export const IDENTIFIER_PLACEHOLDERS: Record = {
'': { grammar: PATH, what: 'bound plan file path' },
'': { grammar: PATH, what: 'preregistration file path' },
'': { grammar: PATH, what: 'results table file path' },
+ '': { grammar: PATH, what: 'hand-label JSONL file path' },
'': { grammar: PATH, what: 'sketch directory printed by an earlier block' },
'': { grammar: PATH, what: 'finalized HTML path' },
'': { grammar: PATH, what: 'saved page HTML path' },
diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts
index 71f78d20b..fa721ce05 100644
--- a/test/pricing-eval-plan.test.ts
+++ b/test/pricing-eval-plan.test.ts
@@ -57,7 +57,9 @@ describe('preregistration', () => {
expect(check.empty.length).toBe(PREREG_SECTIONS.length);
expect(check.ok).toBe(false);
expect(PREREG_SECTIONS).toContain('Comparison');
- const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack');
+ const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack')
+ .replace('## Decisions\n- filled in', '## Decisions\nif the bar is met: run the full set\nif the bar is missed: stop and fix the scorer')
+ .replace('## Scoring\n- filled in', '## Scoring\njudge: none');
expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] });
expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]);
});