From 0fd0977e91579ed24c98818ad364e2e7da363cb7 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 16 Oct 2026 23:15:01 +0000 Subject: [PATCH] feat(eval-plan): W8/W9 tests, docs/eval-models.md, troubleshooting anchors, /eval-plan steps Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> (cherry picked from commit 9335e45de2dd0ba9432482a3d5eee069f22370b6) --- bin/gstack-doctor-components.sh | 5 + bin/gstack-eval-plan | 10 +- docs/eval-models.md | 89 ++++++++++ docs/troubleshooting.md | 64 ++++++++ eval-plan/SKILL.md | 15 ++ eval-plan/SKILL.md.tmpl | 15 ++ test/eval-models.test.ts | 187 ++++++++++++++++++++++ test/eval-plan-prereg-diagnostics.test.ts | 123 ++++++++++++++ 8 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 docs/eval-models.md create mode 100644 test/eval-models.test.ts create mode 100644 test/eval-plan-prereg-diagnostics.test.ts diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 27b372307..8128e94f0 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -234,6 +234,11 @@ gstack_result_anchor() { OUTSIDE_VOICE_UNAVAILABLE) printf '%s' 'outside-voice-unavailable' ;; ARTIFACT_MACHINE_PATH) printf '%s' 'artifact-machine-path' ;; ARTIFACT_PUBLIC_TARGET_EXISTS) printf '%s' 'artifact-public-target-exists' ;; + PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; + EVAL_MODELS_INVALID) printf '%s' 'eval-models-invalid' ;; + EVAL_MODEL_DENIED) printf '%s' 'eval-model-denied' ;; + EVAL_MODEL_OLD_GENERATION) printf '%s' 'eval-model-old-generation' ;; + EVAL_MODEL_GENERATION_UNRESOLVED) printf '%s' 'eval-model-generation-unresolved' ;; *) return 1 ;; esac } diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 649592135..e004c4793 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -22,14 +22,16 @@ * from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the * base branch's .gstack/pregate.json read from the HEAD commit) after checking * its binding (exact gate command, head, input hashes); it executes nothing and - * writes a receipt (docs/eval-calls.md). Exit 0 ok, 1 fail (incomplete + * writes a receipt (docs/eval-calls.md). arms, dry-run, pilot start and + * price-gate lint every arm's model against the base branch's + * .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete * prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused. * Logic: lib/eval-plan.ts. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { checkPrereg, DEFAULT_TOKENS, renderPreregDiagnostics, runPreregDiagnostics, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan'; +import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; @@ -56,6 +58,10 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md + Model rules: arms, dry-run, pilot start and price-gate check every arm against .gstack/eval-models.json from + origin/ (--base b; --models-from head for the unreviewed working-tree copy) at --scope smoke|full + (default full; price-gate uses each call row's scope); docs/eval-models.md + ${exitTableHelp()}`; function flag(args: string[], name: string): string | undefined { diff --git a/docs/eval-models.md b/docs/eval-models.md new file mode 100644 index 000000000..406149101 --- /dev/null +++ b/docs/eval-models.md @@ -0,0 +1,89 @@ +# Eval model rules: `.gstack/eval-models.json` + +A repository that has rules about which models its evals may use (for example +"Fable in smoke tests only", "no gpt-5.4-mini", "the newest model of each +family") writes them as data in `.gstack/eval-models.json`. `gstack-eval-plan` +checks every arm against them: `arms`, `dry-run`, `pilot start`, and every +call row of `price-gate`. gstack never reads these rules from prose in +`CLAUDE.md` or any other document. Plan: field list 2, W9 +(`docs/designs/field-list-2-2026-10-16/PLAN.md`). + +## The file + +```json +{ + "schema_version": 1, + "allow": ["claude-*", "gpt-*"], + "deny": ["gpt-5.4-mini"], + "scopes": { "full": { "deny": ["claude-fable-*"] } }, + "families": { + "claude-opus": ["claude-opus-5-5", "claude-opus-5", "claude-opus-4-8"], + "claude-fable": ["claude-fable-5-1", "claude-fable-5"], + "gpt-sol": ["gpt-6.1-sol", "gpt-6-sol"] + } +} +``` + +| Key | Required | Meaning | +|---|---|---| +| `schema_version` | yes | `1` | +| `allow` | yes | Model ids or globs an arm may use. An id that no entry matches fails, so a misspelled or new model is never admitted by default. Use `["*"]` to allow everything not denied. | +| `deny` | no | Ids or globs no arm may use. Checked first. | +| `scopes` | no | Per eval scope (`smoke` or `full`), an extra `allow` and/or `deny` list. "Fable in smoke only" is `scopes.full.deny: ["claude-fable-*"]`. | +| `families` | no | Family name to literal ids, **newest first**. With this key present, an arm must be the newest id of its family (`EVAL_MODEL_OLD_GENERATION` otherwise), and an arm in no family fails `EVAL_MODEL_GENERATION_UNRESOLVED` instead of passing unchecked. | + +Globs use `*` (any run of characters) and `?` (one character); every other +character is literal. A rule matches the id as written, the id without a +provider prefix (`openai/gpt-5.4-mini` matches `gpt-5.4-mini`), and a Claude id +without its date suffix (`claude-opus-5-5-20261001` matches `claude-opus-5-5`). +An unknown key, an unknown scope, a glob inside `families`, or an id listed in +two families makes the whole file invalid (`EVAL_MODELS_INVALID`), and then no +arm passes. + +The rules apply whatever gstack's own model catalog (`lib/model-catalog.ts`) +holds: an uncatalogued model is allowed or denied by the file alone. The +catalog only checks that `families` is current. If a family lists the +catalog's current frontier or smart model after another id, the file is stale +and invalid. + +## Order of checks + +For each arm at its scope, the first rule that refuses decides: + +1. `deny` → `EVAL_MODEL_DENIED` +2. `scopes..deny` → `EVAL_MODEL_DENIED` +3. no `allow` entry matches → `EVAL_MODEL_DENIED` (unknown ids fail closed) +4. `scopes..allow` is set and does not match → `EVAL_MODEL_DENIED` +5. `families` is set: in no family → `EVAL_MODEL_GENERATION_UNRESOLVED`; not the family's first id → `EVAL_MODEL_OLD_GENERATION` + +## Scope + +`price-gate` checks each call row at that row's `scope` (`GSTACK_EVAL_CALLS v1`, +`docs/eval-calls.md`). `arms`, `dry-run` and `pilot start` take +`--scope smoke|full`, which defaults to `full`. So a smoke-only model fails +unless the command says it is a smoke run. + +## Which copy is read + +By default the reviewed copy is read: `git show origin/:.gstack/eval-models.json` +(`--base`, default origin/HEAD's branch). A branch cannot loosen the rules it +is checked against. When the working-tree copy differs, an +`EVAL_MODELS_NOTE:` line says so. `--models-from head` lints with the working-tree copy +instead. Use it for the PR that adds or changes the file; the source label +says `head (unreviewed)`. When neither copy exists, nothing is checked and +the line says `EVAL_MODELS: none ...`. When the base cannot be resolved but a +working-tree copy exists, the command refuses (`EVAL_MODELS_INVALID`) rather +than skipping the rules. + +## Output + +``` +EVAL_MODEL: claude-fable-5-1 scope=full denied (scopes.full.deny claude-fable-*) where=dry-run --arms +EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=dry-run --arms +EVAL_MODELS: fail refused=1/2 source=origin/main@0123456789ab:.gstack/eval-models.json +``` + +Each refusal also prints a result line on stderr ending in its code. `dry-run` +records the check as `checks.model_rules` in `dry-run.json` and fails +`DRY_RUN_RESULT`. `pilot start` reserves nothing. `price-gate` writes no +receipt. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index bc046c2ab..487156ad8 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3366,3 +3366,67 @@ them. **Fix.** Pass an empty or new directory, or the directory that already holds this run's public copy (it is replaced as a whole). + + +### `PREREG_DIAGNOSTIC_UNTESTED` + +**Meaning.** `gstack-eval-plan prereg --check --allow-repo-commands` ran +the preregistration's declared summarizer (`## Diagnostics`: `summarizer:`, +`fixture:` and one `| diagnostic | expected |` row each) on its synthetic +fixture. Its result did not match the preregistered definitions: a count +differed (`mismatch`), a declared diagnostic was absent (`missing`), it +reported one nobody preregistered (`undeclared`), or the summarizer failed, +timed out, or did not end its stdout with one JSON object of counts. Each +`PREREG_DIAGNOSTIC:` line names one diagnostic. This is the gbrain-evals #162 +(D7) shape, where two diagnostic counts needed a post-paid amendment. + +**Fix.** Fix the summarizer, or correct the definition (the expected count or +the fixture), until every line says `ok`. Then commit the freeze. The +`PREREG_DIAGNOSTICS:` line's `sha256` binds the fixture you tested. Without +`--allow-repo-commands`, the command lists the summarizer +(`PREREG_COMMAND:`) and exits 3 (`REPO_COMMANDS_NOT_ALLOWED`), because the +summarizer is a repository command. + + +### `EVAL_MODELS_INVALID` + +**Meaning.** `.gstack/eval-models.json` could not be read or failed +validation. Causes include JSON errors, `schema_version` other than 1, a +missing or empty `allow`, an unknown key or scope, a glob in `families`, an id +in two families, or a family that lists gstack's current catalog model after an +older id. It also fires when `origin/` cannot be resolved while a +working-tree copy exists. While the rules are invalid, no arm passes. + +**Fix.** Correct the file as the line says (`docs/eval-models.md`) in a +reviewed PR, or fetch the base branch. To test an unmerged edit, use +`--models-from head`. + + +### `EVAL_MODEL_DENIED` + +**Meaning.** An arm's model matched `deny` or its scope's `deny`, matched no +`allow` entry (unknown ids fail closed), or matched no entry of its scope's +`allow` list. The `EVAL_MODEL:` line names the rule. + +**Fix.** Pick a model the rules allow for that scope. For a smoke run, pass +`--scope smoke` (`price-gate` uses each call row's scope). To change the rule, +edit `.gstack/eval-models.json` in a reviewed PR. + + +### `EVAL_MODEL_OLD_GENERATION` + +**Meaning.** The rules declare `families` (newest first), and an arm uses an id +that is not the first one in its family. + +**Fix.** Use the family's newest id, which the `EVAL_MODEL:` line names. If the +order is wrong, fix `families` in a reviewed PR. + + +### `EVAL_MODEL_GENERATION_UNRESOLVED` + +**Meaning.** The rules declare `families`, so every arm must be its family's +newest generation. This arm's model is in no family, so its generation cannot be +checked, and the arm fails instead of passing unchecked. + +**Fix.** Add the model to its family (newest first) in +`.gstack/eval-models.json` in a reviewed PR, or use a model a family lists. diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 2e8d741c0..79d793ad9 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -399,6 +399,15 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +When the summarizer counts diagnostics, declare them before the freeze commit in a +`## Diagnostics` section: `summarizer: `, `fixture: `, and one `| diagnostic | expected |` row per diagnostic, with the count the +fixture is built to produce. The summarizer's last stdout line is one JSON object of counts. +Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is +listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`. +`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an +undeclared one does not match its definition. Fix that before you freeze. + ### Step 1b: Arms and their price ```bash @@ -412,6 +421,12 @@ and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing page or drop the arm. Copy the priced arms into the Arms section. +When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families +newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and +`price-gate` print one `EVAL_MODEL:` line per arm. Pass `--scope smoke` for a smoke run; +the default is `full`. A refused arm (`EVAL_MODEL_DENIED`, `EVAL_MODEL_OLD_GENERATION`, +`EVAL_MODEL_GENERATION_UNRESOLVED`) is replaced, never argued past. + ## Step 2: zero-cost stub-model dry run Run the pipeline over the real development rows with a stub model that costs nothing: diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 2dcf76aed..c541639ec 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -71,6 +71,15 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +When the summarizer counts diagnostics, declare them before the freeze commit in a +`## Diagnostics` section: `summarizer: `, `fixture: `, and one `| diagnostic | expected |` row per diagnostic, with the count the +fixture is built to produce. The summarizer's last stdout line is one JSON object of counts. +Then run `prereg --check --allow-repo-commands`. Without the flag, the summarizer is +listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`. +`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an +undeclared one does not match its definition. Fix that before you freeze. + ### Step 1b: Arms and their price ```bash @@ -84,6 +93,12 @@ and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing page or drop the arm. Copy the priced arms into the Arms section. +When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families +newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and +`price-gate` print one `EVAL_MODEL:` line per arm. Pass `--scope smoke` for a smoke run; +the default is `full`. A refused arm (`EVAL_MODEL_DENIED`, `EVAL_MODEL_OLD_GENERATION`, +`EVAL_MODEL_GENERATION_UNRESOLVED`) is replaced, never argued past. + ## Step 2: zero-cost stub-model dry run Run the pipeline over the real development rows with a stub model that costs nothing: diff --git a/test/eval-models.test.ts b/test/eval-models.test.ts new file mode 100644 index 000000000..ed243a88c --- /dev/null +++ b/test/eval-models.test.ts @@ -0,0 +1,187 @@ +/** + * Field list 2, W9: .gstack/eval-models.json and the lint gstack-eval-plan runs + * over every arm (arms, dry-run, pilot start, price-gate). gbrain's rules + * ("Fable in smoke only", "no gpt-5.4-mini", "the newest model of each family") + * as data; literal allow/deny independent of MODEL_CATALOG; unknown ids fail + * closed; the reviewed copy comes from origin/. + */ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { formatEvalCalls } from '../lib/eval-plan'; +import { lintArmModels, parseEvalModels, type ArmUse, type EvalModelRules } from '../lib/eval-models'; +import { capyHost, type CapyHost } from './helpers/capy-host'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const GBRAIN_RULES = { + schema_version: 1, + allow: ['claude-*', 'gpt-*'], + deny: ['gpt-5.4-mini'], + scopes: { full: { deny: ['claude-fable-*'] } }, + families: { + 'claude-opus': ['claude-opus-5-5', 'claude-opus-5', 'claude-opus-4-8'], + 'claude-fable': ['claude-fable-5-1', 'claude-fable-5'], + 'gpt-sol': ['gpt-6.1-sol', 'gpt-6-sol'], + }, +}; + +function rules(doc: unknown): EvalModelRules { + const parsed = parseEvalModels(JSON.stringify(doc)); + if ('errors' in parsed) throw new Error(parsed.errors.join('; ')); + return parsed.rules; +} +const verdicts = (r: EvalModelRules, uses: Array<[string, ArmUse['scope']]>) => lintArmModels(r, uses.map(([model, scope]) => ({ model, scope, where: 't' }))).map(f => `${f.model}/${f.scope}: ${f.verdict} (${f.reason})`); + +describe('lintArmModels', () => { + test('gbrain\'s rules: Fable smoke-only, gpt-5.4-mini denied in any form, older generations refused, unknown ids fail closed', () => { + expect(verdicts(rules(GBRAIN_RULES), [ + ['claude-fable-5-1', 'smoke'], ['claude-fable-5-1', 'full'], ['gpt-5.4-mini', 'smoke'], ['openai/gpt-5.4-mini', 'full'], + ['claude-opus-4-8', 'full'], ['anthropic/claude-opus-5-5-20261001', 'full'], ['gpt-6.1-sol', 'full'], ['gemini-3.8-flash', 'smoke'], ['claude-haiku-5-5', 'full'], + ])).toEqual([ + 'claude-fable-5-1/smoke: ok (allow claude-*; newest in families.claude-fable)', + 'claude-fable-5-1/full: denied (scopes.full.deny claude-fable-*)', + 'gpt-5.4-mini/smoke: denied (deny gpt-5.4-mini)', + 'openai/gpt-5.4-mini/full: denied (deny gpt-5.4-mini)', + 'claude-opus-4-8/full: old-generation (families.claude-opus newest is claude-opus-5-5)', + 'anthropic/claude-opus-5-5-20261001/full: ok (allow claude-*; newest in families.claude-opus)', + 'gpt-6.1-sol/full: ok (allow gpt-*; newest in families.gpt-sol)', + 'gemini-3.8-flash/smoke: denied (matches no allow entry (unknown ids fail closed))', + 'claude-haiku-5-5/full: generation-unresolved (in no families entry, so the newest-generation rule cannot be checked)', + ]); + }); + + test('literal allow/deny ignore catalog membership: uncatalogued ids are allowed or denied by the file alone; scope allow lists narrow', () => { + const r = rules({ schema_version: 1, allow: ['voyage-4-large', 'claude-sonnet-5-5', 'claude-opus-5-5'], deny: ['claude-opus-5-5'], scopes: { smoke: { allow: ['claude-sonnet-*'] } } }); + expect(verdicts(r, [['voyage-4-large', 'full'], ['claude-opus-5-5', 'full'], ['claude-sonnet-5-5', 'smoke'], ['voyage-4-large', 'smoke']])).toEqual([ + 'voyage-4-large/full: ok (allow voyage-4-large)', + 'claude-opus-5-5/full: denied (deny claude-opus-5-5)', + 'claude-sonnet-5-5/smoke: ok (allow claude-sonnet-5-5)', + 'voyage-4-large/smoke: denied (matches no scopes.smoke.allow entry)', + ]); + }); +}); + +describe('parseEvalModels', () => { + test('the file fails closed on a missing allow, unknown keys and scopes, globs in families, duplicates and a malformed id', () => { + const errors = (doc: unknown) => { const p = parseEvalModels(JSON.stringify(doc)); return 'errors' in p ? p.errors : []; }; + expect(errors({ schema_version: 1 })).toEqual(['allow must be a non-empty array of model ids or globs']); + expect(errors({ schema_version: 2, allow: ['*'], newest: true })).toEqual(['unknown key "newest" (known: schema_version, allow, deny, scopes, families)', 'schema_version must be 1']); + expect(errors({ schema_version: 1, allow: ['*'], scopes: { release: { deny: [] }, smoke: { only: [] } } })).toEqual(['scopes.release: unknown eval scope (known: smoke, full)', 'scopes.smoke: unknown key(s) only (known: allow, deny)']); + expect(errors({ schema_version: 1, allow: ['*'], families: { a: ['claude-*'], b: ['x-1', 'x-2'], c: ['x-2'] } })).toEqual([ + 'families.a[0] "claude-*" is not a model id (family entries are literal ids, newest first)', 'families: x-2 is listed in both b and c', + ]); + expect(errors({ schema_version: 1, allow: ['claude-[a-z]+'] })[0]).toContain('allow[0] "claude-[a-z]+" is not a model id or glob'); + expect(parseEvalModels('{ allow: [] }')).toMatchObject({ errors: [expect.stringContaining('not valid JSON')] }); + }); + + test('MODEL_CATALOG checks only currency: a family listing gstack\'s current smart model after an older id is stale', () => { + const p = parseEvalModels(JSON.stringify({ schema_version: 1, allow: ['*'], families: { 'claude-opus': ['claude-opus-5', 'claude-opus-5-5'] } })); + expect('errors' in p && p.errors).toEqual([expect.stringMatching(/^families\.claude-opus lists claude-opus-5 before claude-opus-5-5, gstack's current anthropic smart model \(lib\/model-catalog\.ts, verified \d{4}-\d{2}-\d{2}\); list newest first$/)]); + expect('rules' in parseEvalModels(JSON.stringify(GBRAIN_RULES))).toBe(true); + }); +}); + +describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)', () => { + let host: CapyHost; + let repo: string; + const git = (...args: string[]) => { + const r = spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 }); + if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); + return r.stdout.trim(); + }; + const run = (args: string[]) => { + const r = spawnSync(BIN, args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; + }; + function seed(doc: unknown | null) { + const origin = path.join(host.base, 'origin.git'); + spawnSync('git', ['init', '-q', '--bare', '-b', 'main', origin], { env: host.env, timeout: 30_000 }); + repo = path.join(host.base, 'repo'); + fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true }); + git('init', '-q', '-b', 'main'); + git('remote', 'add', 'origin', origin); + fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n'); + fs.writeFileSync(path.join(repo, 'retry.txt'), 'Answer again in the required JSON shape.\n'); + if (doc) fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify(doc, null, 2) + '\n'); + git('add', '-A'); git('commit', '-q', '-m', 'seed'); git('push', '-q', 'origin', 'main'); + } + beforeEach(() => { host = capyHost(); }); + afterEach(() => host.cleanup()); + + test('arms: the reviewed rules from origin/main refuse a denied arm; a working-tree edit does not loosen them unless --models-from head', () => { + seed(GBRAIN_RULES); + const r = run(['arms', '--arms', 'claude-opus-5-5,gpt-5.4-mini']); + expect(r.code).toBe(1); + expect(r.out).toContain('EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=flag:--arms'); + expect(r.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full denied (deny gpt-5.4-mini) where=flag:--arms'); + expect(r.out).toMatch(/EVAL_MODELS: fail refused=1\/2 source=origin\/main@[0-9a-f]{12}:\.gstack\/eval-models\.json/); + expect(r.err).toContain('(EVAL_MODEL_DENIED)'); + fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ ...GBRAIN_RULES, deny: [] })); + const edited = run(['arms', '--arms', 'gpt-5.4-mini']); + expect(edited.code).toBe(1); + expect(edited.out).toContain('EVAL_MODELS_NOTE: the working-tree .gstack/eval-models.json differs from the reviewed copy; linting with origin/main'); + const head = run(['arms', '--arms', 'gpt-5.4-mini', '--models-from', 'head']); + expect(head.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full generation-unresolved'); + expect(head.out).toContain('source=head (unreviewed) .gstack/eval-models.json'); + expect(head.err).toContain('(EVAL_MODEL_GENERATION_UNRESOLVED)'); + }); + + test('dry-run and pilot start: Fable fails the full scope (no reservation), passes --scope smoke; the dry-run record carries the check', () => { + seed(GBRAIN_RULES); + const out = path.join(host.base, 'out'); + const base = ['dry-run', '--input', 'rows.jsonl', '--arms', 'claude-fable-5-1', '--retry-prompt', 'retry.txt', '--out', out]; + const full = run(base); + expect(full.code).toBe(1); + expect(full.out).toContain('CHECK: model-rules refused claude-fable-5-1 (scope full, dry-run --arms): scopes.full.deny claude-fable-*'); + expect(full.out).toContain('DRY_RUN_RESULT: fail'); + const smoke = run([...base, '--scope', 'smoke']); + expect(smoke.code).toBe(0); + expect(smoke.out).toMatch(/CHECK: model-rules ok \(source origin\/main@[0-9a-f]{12}:\.gstack\/eval-models\.json\)/); + expect(JSON.parse(fs.readFileSync(path.join(out, 'dry-run.json'), 'utf8')).checks.model_rules).toMatchObject({ ok: true, refused: [] }); + const pilotOut = path.join(host.base, 'pilot'); + const pilot = run(['pilot', 'start', '--out', pilotOut, '--cap', '5', '--arms', 'claude-fable-5-1', '--dry-run', path.join(out, 'dry-run.json')]); + expect(pilot.code).toBe(1); + expect(pilot.err).toContain('(EVAL_MODEL_DENIED)'); + expect(fs.existsSync(path.join(pilotOut, 'spend.json'))).toBe(false); + expect(run(['pilot', 'start', '--out', pilotOut, '--cap', '5', '--arms', 'claude-fable-5-1', '--dry-run', path.join(out, 'dry-run.json'), '--scope', 'smoke']).code).toBe(0); + expect(run([...base, '--scope', 'release']).code).toBe(2); + }); + + test('price-gate lints each call row at its own scope and writes no receipt on a refusal', () => { + seed(GBRAIN_RULES); + const head = git('rev-parse', 'HEAD'); + const gate = 'bun run eval:smoke'; + const plan = path.join(host.base, 'plan.calls'); + const rows = [ + { purpose: 'smoke', model: 'claude-fable-5-1', calls: 3, in_tokens: 3_000, out_tokens: 1_000, max_out_tokens: 8_192, scope: 'smoke' as const }, + { purpose: 'judge', model: 'claude-opus-4-8', calls: 3, in_tokens: 3_000, out_tokens: 1_000, max_out_tokens: 2_048, scope: 'full' as const }, + ]; + fs.writeFileSync(plan, formatEvalCalls({ gate, head, inputs: [], preconditions: [], rows })); + const receipts = path.join(host.base, 'receipts'); + const r = run(['price-gate', '--gate', gate, '--plan-file', plan, '--out', receipts]); + expect(r.code).toBe(1); + expect(r.out).toContain('EVAL_MODEL: claude-fable-5-1 scope=smoke ok'); + expect(r.out).toContain('EVAL_MODEL: claude-opus-4-8 scope=full old-generation (families.claude-opus newest is claude-opus-5-5) where=purpose=judge'); + expect(r.err).toContain('(EVAL_MODEL_OLD_GENERATION)'); + expect(fs.existsSync(receipts)).toBe(false); + fs.writeFileSync(plan, formatEvalCalls({ gate, head, inputs: [], preconditions: [], rows: [rows[0]!] })); + const ok = run(['price-gate', '--gate', gate, '--plan-file', plan, '--out', receipts]); + expect(ok.code).toBe(0); + expect(fs.readdirSync(receipts).length).toBe(1); + }); + + test('no rules file changes nothing; an invalid reviewed file fails EVAL_MODELS_INVALID for every arm', () => { + seed(null); + const none = run(['arms', '--arms', 'gpt-5.4-mini']); + expect(none.code).toBe(0); + expect(none.out).toMatch(/EVAL_MODELS: none at origin\/main@[0-9a-f]{12}\n/); + fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ schema_version: 1, allow: ['claude-*'], deny: 'gpt-5.4-mini' })); + git('add', '-A'); git('commit', '-q', '-m', 'rules'); git('push', '-q', 'origin', 'main'); + const invalid = run(['arms', '--arms', 'claude-opus-5-5']); + expect(invalid.code).toBe(1); + expect(invalid.out).toMatch(/EVAL_MODELS: invalid source=origin\/main@/); + expect(invalid.err).toContain('deny must be an array of model ids or globs'); + expect(invalid.err).toContain('(EVAL_MODELS_INVALID)'); + }); +}); diff --git a/test/eval-plan-prereg-diagnostics.test.ts b/test/eval-plan-prereg-diagnostics.test.ts new file mode 100644 index 000000000..7271cc7a8 --- /dev/null +++ b/test/eval-plan-prereg-diagnostics.test.ts @@ -0,0 +1,123 @@ +/** + * Field list 2, W8: `gstack-eval-plan prereg --check` runs a preregistration's + * declared summarizer on its declared synthetic fixture (only with + * --allow-repo-commands) and fails PREREG_DIAGNOSTIC_UNTESTED when a count + * differs from its preregistered definition, before the freeze commit. The D7 + * shape (gbrain-evals #162): two diagnostic counts that did not match. + */ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval-plan'; +import { capyHost, type CapyHost } from './helpers/capy-host'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const SUMMARIZER = [ + 'touch ran.marker', + 'r=$(grep -c \'"kind":"refusal"\' "$1")', + 't=$(grep -c \'"kind":"truncated"\' "$1")', + 'echo "summarizing $1"', + 'printf \'{"refusals":%s,"truncated":%s}\\n\' "$r" "$t"', +].join('\n') + '\n'; +const RECORDS = ['refusal', 'refusal', 'truncated', 'answer'].map(kind => JSON.stringify({ kind })).join('\n') + '\n'; + +function diagnostics(rows: Array<[string, string]>, o: { summarizer?: string; fixture?: string } = {}): string { + return `\n## Diagnostics\nsummarizer: ${o.summarizer ?? '`sh summarize.sh {fixture}`'}\nfixture: ${o.fixture ?? 'fixtures/synthetic records.jsonl'}\n\n| diagnostic | expected |\n|---|---|\n${rows.map(([n, c]) => `| ${n} | ${c} |`).join('\n')}\n`; +} + +let host: CapyHost; +let repo: string; +beforeEach(() => { + host = capyHost(); + repo = path.join(host.base, 'repo'); + fs.mkdirSync(path.join(repo, 'fixtures'), { recursive: true }); + const init = spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 }); + if (init.status !== 0) throw new Error(init.stderr); + fs.writeFileSync(path.join(repo, 'summarize.sh'), SUMMARIZER); + fs.writeFileSync(path.join(repo, 'fixtures', 'synthetic records.jsonl'), RECORDS); + fs.mkdirSync(path.join(repo, 'evals')); +}); +afterEach(() => host.cleanup()); + +function check(text: string, ...flags: string[]) { + const file = path.join(repo, 'evals', 'prereg.md'); + fs.writeFileSync(file, text); + const r = spawnSync(BIN, ['prereg', '--check', file, ...flags], { cwd: host.base, env: host.env, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '', ran: fs.existsSync(path.join(repo, 'ran.marker')) }; +} + +describe('prereg --check: the Diagnostics section', () => { + test('the declared summarizer is listed, not run, without --allow-repo-commands (exit 3)', () => { + const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']])); + expect(r.code).toBe(3); + expect(r.ran).toBe(false); + expect(r.out).toContain('PREREG: ok present=5/5 diagnostics=2 file='); + expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`); + expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)'); + expect(r.out).not.toContain('PREREG_DIAGNOSTICS:'); + }); + + test('with --allow-repo-commands, matching counts pass and bind the fixture by sha256; the path reaches the command quoted', () => { + const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]), '--allow-repo-commands'); + expect(r.err).toBe(''); + expect(r.code).toBe(0); + expect(r.ran).toBe(true); + const sha = createHash('sha256').update(RECORDS).digest('hex').slice(0, 12); + expect(r.out).toContain('PREREG_DIAGNOSTIC: refusals expected=2 got=2 ok\nPREREG_DIAGNOSTIC: truncated expected=1 got=1 ok\n'); + expect(r.out).toContain(`PREREG_DIAGNOSTICS: ok n=2 fixture=fixtures/synthetic records.jsonl sha256=${sha}\n`); + }); + + test('the D7 shape: two counts off their definitions, a declared count the summarizer never reports and an undeclared one all fail PREREG_DIAGNOSTIC_UNTESTED', () => { + const r = check(FILLED + diagnostics([['refusals', '3'], ['truncated', '0'], ['empty_answers', '0']]), '--allow-repo-commands'); + expect(r.code).toBe(1); + expect(r.out).toContain('PREREG_DIAGNOSTIC: refusals expected=3 got=2 mismatch'); + expect(r.out).toContain('PREREG_DIAGNOSTIC: truncated expected=0 got=1 mismatch'); + expect(r.out).toContain('PREREG_DIAGNOSTIC: empty_answers expected=0 got=none missing'); + expect(r.out).toContain('PREREG_DIAGNOSTICS: untested n=3'); + expect(r.err).toContain('refusals mismatch (expected 3, got 2); truncated mismatch (expected 0, got 1); empty_answers missing (expected 0, got none)'); + expect(r.err.trim()).toEndWith('(PREREG_DIAGNOSTIC_UNTESTED)'); + const undeclared = check(FILLED + diagnostics([['refusals', '2']]), '--allow-repo-commands'); + expect(undeclared.code).toBe(1); + expect(undeclared.out).toContain('PREREG_DIAGNOSTIC: truncated expected=undeclared got=1 undeclared'); + expect(undeclared.err).toContain('(PREREG_DIAGNOSTIC_UNTESTED)'); + }); + + test('a failing summarizer, non-JSON output and a missing fixture are untested, never passed', () => { + fs.writeFileSync(path.join(repo, 'fail.sh'), 'echo "boom: no such column" >&2\nexit 2\n'); + const failed = check(FILLED + diagnostics([['refusals', '2']], { summarizer: 'sh fail.sh {fixture}' }), '--allow-repo-commands'); + expect(failed.code).toBe(1); + expect(failed.out).toContain('PREREG_DIAGNOSTICS: untested n=0'); + expect(failed.err).toContain('summarizer exited 2: boom: no such column'); + const prose = check(FILLED + diagnostics([['refusals', '2']], { summarizer: 'echo refusals=2 {fixture}' }), '--allow-repo-commands'); + expect(prose.code).toBe(1); + expect(prose.err).toContain('not a JSON object of diagnostic counts'); + const missing = check(FILLED + diagnostics([['refusals', '2']], { fixture: 'fixtures/nope.jsonl' }), '--allow-repo-commands'); + expect(missing.code).toBe(1); + expect(missing.ran).toBe(false); + expect(missing.err).toContain('declared fixture fixtures/nope.jsonl is not a file'); + }); + + test('an incomplete declaration is PREREG_INCOMPLETE and never runs, even with --allow-repo-commands', () => { + const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands'); + expect(r.code).toBe(1); + expect(r.ran).toBe(false); + expect(r.out).toContain('PREREG: incomplete present=5/5 diagnostics=invalid'); + expect(r.err).toContain('diagnostics: row "| refusals | two |"'); + expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})'); + expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows'); + expect(r.err.trim()).toEndWith('(PREREG_INCOMPLETE)'); + }); + + test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => { + const r = check(FILLED); + expect(r.code).toBe(0); + expect(r.out).toBe(`PREREG: ok present=5/5 file=${path.join(repo, 'evals', 'prereg.md')}\n`); + expect(parsePreregDiagnostics(FILLED)).toBeNull(); + expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n\n')).toBeNull(); + expect(preregTemplate()).toContain('## Diagnostics'); + expect(checkPrereg(preregTemplate()).diagnostics).toBeNull(); + }); +});