mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 11:10:13 +01:00
feat(eval-plan): W8/W9 tests, docs/eval-models.md, troubleshooting anchors, /eval-plan steps
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
(cherry picked from commit 9335e45de2)
This commit is contained in:
garrytan committed
1 parent
c390f282d2
commit
0fd0977e91
8 files changed
+506
-2
No files matched your search
@@ -234,6 +234,11 @@ gstack_result_anchor() {
|
||||
OUTSIDE_VOICE_UNAVAILABLE) printf '%s' 'outside-voice-unavailable' ;;
|
||||
ARTIFACT_MACHINE_PATH) printf '%s' 'artifact-machine-path' ;;
|
||||
ARTIFACT_PUBLIC_TARGET_EXISTS) printf '%s' 'artifact-public-target-exists' ;;
|
||||
PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;;
|
||||
EVAL_MODELS_INVALID) printf '%s' 'eval-models-invalid' ;;
|
||||
EVAL_MODEL_DENIED) printf '%s' 'eval-model-denied' ;;
|
||||
EVAL_MODEL_OLD_GENERATION) printf '%s' 'eval-model-old-generation' ;;
|
||||
EVAL_MODEL_GENERATION_UNRESOLVED) printf '%s' 'eval-model-generation-unresolved' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -22,14 +22,16 @@
|
||||
* from the repo's call-plan DATA file (`--plan-file`, else `price_file` in the
|
||||
* base branch's .gstack/pregate.json read from the HEAD commit) after checking
|
||||
* its binding (exact gate command, head, input hashes); it executes nothing and
|
||||
* writes a receipt (docs/eval-calls.md). Exit 0 ok, 1 fail (incomplete
|
||||
* writes a receipt (docs/eval-calls.md). arms, dry-run, pilot start and
|
||||
* price-gate lint every arm's model against the base branch's
|
||||
* .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete
|
||||
* prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused.
|
||||
* Logic: lib/eval-plan.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { checkPrereg, DEFAULT_TOKENS, renderPreregDiagnostics, runPreregDiagnostics, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, resolveArms, resolvePlanSource, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
@@ -56,6 +58,10 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
|
||||
policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
|
||||
receipt (default .gstack/tmp/price-gate/); docs/eval-calls.md
|
||||
|
||||
Model rules: arms, dry-run, pilot start and price-gate check every arm against .gstack/eval-models.json from
|
||||
origin/<base> (--base b; --models-from head for the unreviewed working-tree copy) at --scope smoke|full
|
||||
(default full; price-gate uses each call row's scope); docs/eval-models.md
|
||||
|
||||
${exitTableHelp()}`;
|
||||
|
||||
function flag(args: string[], name: string): string | undefined {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Eval model rules: `.gstack/eval-models.json`
|
||||
|
||||
A repository that has rules about which models its evals may use (for example
|
||||
"Fable in smoke tests only", "no gpt-5.4-mini", "the newest model of each
|
||||
family") writes them as data in `.gstack/eval-models.json`. `gstack-eval-plan`
|
||||
checks every arm against them: `arms`, `dry-run`, `pilot start`, and every
|
||||
call row of `price-gate`. gstack never reads these rules from prose in
|
||||
`CLAUDE.md` or any other document. Plan: field list 2, W9
|
||||
(`docs/designs/field-list-2-2026-10-16/PLAN.md`).
|
||||
|
||||
## The file
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"allow": ["claude-*", "gpt-*"],
|
||||
"deny": ["gpt-5.4-mini"],
|
||||
"scopes": { "full": { "deny": ["claude-fable-*"] } },
|
||||
"families": {
|
||||
"claude-opus": ["claude-opus-5-5", "claude-opus-5", "claude-opus-4-8"],
|
||||
"claude-fable": ["claude-fable-5-1", "claude-fable-5"],
|
||||
"gpt-sol": ["gpt-6.1-sol", "gpt-6-sol"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Key | Required | Meaning |
|
||||
|---|---|---|
|
||||
| `schema_version` | yes | `1` |
|
||||
| `allow` | yes | Model ids or globs an arm may use. An id that no entry matches fails, so a misspelled or new model is never admitted by default. Use `["*"]` to allow everything not denied. |
|
||||
| `deny` | no | Ids or globs no arm may use. Checked first. |
|
||||
| `scopes` | no | Per eval scope (`smoke` or `full`), an extra `allow` and/or `deny` list. "Fable in smoke only" is `scopes.full.deny: ["claude-fable-*"]`. |
|
||||
| `families` | no | Family name to literal ids, **newest first**. With this key present, an arm must be the newest id of its family (`EVAL_MODEL_OLD_GENERATION` otherwise), and an arm in no family fails `EVAL_MODEL_GENERATION_UNRESOLVED` instead of passing unchecked. |
|
||||
|
||||
Globs use `*` (any run of characters) and `?` (one character); every other
|
||||
character is literal. A rule matches the id as written, the id without a
|
||||
provider prefix (`openai/gpt-5.4-mini` matches `gpt-5.4-mini`), and a Claude id
|
||||
without its date suffix (`claude-opus-5-5-20261001` matches `claude-opus-5-5`).
|
||||
An unknown key, an unknown scope, a glob inside `families`, or an id listed in
|
||||
two families makes the whole file invalid (`EVAL_MODELS_INVALID`), and then no
|
||||
arm passes.
|
||||
|
||||
The rules apply whatever gstack's own model catalog (`lib/model-catalog.ts`)
|
||||
holds: an uncatalogued model is allowed or denied by the file alone. The
|
||||
catalog only checks that `families` is current. If a family lists the
|
||||
catalog's current frontier or smart model after another id, the file is stale
|
||||
and invalid.
|
||||
|
||||
## Order of checks
|
||||
|
||||
For each arm at its scope, the first rule that refuses decides:
|
||||
|
||||
1. `deny` → `EVAL_MODEL_DENIED`
|
||||
2. `scopes.<scope>.deny` → `EVAL_MODEL_DENIED`
|
||||
3. no `allow` entry matches → `EVAL_MODEL_DENIED` (unknown ids fail closed)
|
||||
4. `scopes.<scope>.allow` is set and does not match → `EVAL_MODEL_DENIED`
|
||||
5. `families` is set: in no family → `EVAL_MODEL_GENERATION_UNRESOLVED`; not the family's first id → `EVAL_MODEL_OLD_GENERATION`
|
||||
|
||||
## Scope
|
||||
|
||||
`price-gate` checks each call row at that row's `scope` (`GSTACK_EVAL_CALLS v1`,
|
||||
`docs/eval-calls.md`). `arms`, `dry-run` and `pilot start` take
|
||||
`--scope smoke|full`, which defaults to `full`. So a smoke-only model fails
|
||||
unless the command says it is a smoke run.
|
||||
|
||||
## Which copy is read
|
||||
|
||||
By default the reviewed copy is read: `git show origin/<base>:.gstack/eval-models.json`
|
||||
(`--base`, default origin/HEAD's branch). A branch cannot loosen the rules it
|
||||
is checked against. When the working-tree copy differs, an
|
||||
`EVAL_MODELS_NOTE:` line says so. `--models-from head` lints with the working-tree copy
|
||||
instead. Use it for the PR that adds or changes the file; the source label
|
||||
says `head (unreviewed)`. When neither copy exists, nothing is checked and
|
||||
the line says `EVAL_MODELS: none ...`. When the base cannot be resolved but a
|
||||
working-tree copy exists, the command refuses (`EVAL_MODELS_INVALID`) rather
|
||||
than skipping the rules.
|
||||
|
||||
## Output
|
||||
|
||||
```
|
||||
EVAL_MODEL: claude-fable-5-1 scope=full denied (scopes.full.deny claude-fable-*) where=dry-run --arms
|
||||
EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=dry-run --arms
|
||||
EVAL_MODELS: fail refused=1/2 source=origin/main@0123456789ab:.gstack/eval-models.json
|
||||
```
|
||||
|
||||
Each refusal also prints a result line on stderr ending in its code. `dry-run`
|
||||
records the check as `checks.model_rules` in `dry-run.json` and fails
|
||||
`DRY_RUN_RESULT`. `pilot start` reserves nothing. `price-gate` writes no
|
||||
receipt.
|
||||
@@ -3366,3 +3366,67 @@ them.
|
||||
|
||||
**Fix.** Pass an empty or new directory, or the directory that already holds
|
||||
this run's public copy (it is replaced as a whole).
|
||||
|
||||
<a id="prereg-diagnostic-untested"></a>
|
||||
### `PREREG_DIAGNOSTIC_UNTESTED`
|
||||
|
||||
**Meaning.** `gstack-eval-plan prereg --check <file> --allow-repo-commands` ran
|
||||
the preregistration's declared summarizer (`## Diagnostics`: `summarizer:`,
|
||||
`fixture:` and one `| diagnostic | expected |` row each) on its synthetic
|
||||
fixture. Its result did not match the preregistered definitions: a count
|
||||
differed (`mismatch`), a declared diagnostic was absent (`missing`), it
|
||||
reported one nobody preregistered (`undeclared`), or the summarizer failed,
|
||||
timed out, or did not end its stdout with one JSON object of counts. Each
|
||||
`PREREG_DIAGNOSTIC:` line names one diagnostic. This is the gbrain-evals #162
|
||||
(D7) shape, where two diagnostic counts needed a post-paid amendment.
|
||||
|
||||
**Fix.** Fix the summarizer, or correct the definition (the expected count or
|
||||
the fixture), until every line says `ok`. Then commit the freeze. The
|
||||
`PREREG_DIAGNOSTICS:` line's `sha256` binds the fixture you tested. Without
|
||||
`--allow-repo-commands`, the command lists the summarizer
|
||||
(`PREREG_COMMAND:`) and exits 3 (`REPO_COMMANDS_NOT_ALLOWED`), because the
|
||||
summarizer is a repository command.
|
||||
|
||||
<a id="eval-models-invalid"></a>
|
||||
### `EVAL_MODELS_INVALID`
|
||||
|
||||
**Meaning.** `.gstack/eval-models.json` could not be read or failed
|
||||
validation. Causes include JSON errors, `schema_version` other than 1, a
|
||||
missing or empty `allow`, an unknown key or scope, a glob in `families`, an id
|
||||
in two families, or a family that lists gstack's current catalog model after an
|
||||
older id. It also fires when `origin/<base>` cannot be resolved while a
|
||||
working-tree copy exists. While the rules are invalid, no arm passes.
|
||||
|
||||
**Fix.** Correct the file as the line says (`docs/eval-models.md`) in a
|
||||
reviewed PR, or fetch the base branch. To test an unmerged edit, use
|
||||
`--models-from head`.
|
||||
|
||||
<a id="eval-model-denied"></a>
|
||||
### `EVAL_MODEL_DENIED`
|
||||
|
||||
**Meaning.** An arm's model matched `deny` or its scope's `deny`, matched no
|
||||
`allow` entry (unknown ids fail closed), or matched no entry of its scope's
|
||||
`allow` list. The `EVAL_MODEL:` line names the rule.
|
||||
|
||||
**Fix.** Pick a model the rules allow for that scope. For a smoke run, pass
|
||||
`--scope smoke` (`price-gate` uses each call row's scope). To change the rule,
|
||||
edit `.gstack/eval-models.json` in a reviewed PR.
|
||||
|
||||
<a id="eval-model-old-generation"></a>
|
||||
### `EVAL_MODEL_OLD_GENERATION`
|
||||
|
||||
**Meaning.** The rules declare `families` (newest first), and an arm uses an id
|
||||
that is not the first one in its family.
|
||||
|
||||
**Fix.** Use the family's newest id, which the `EVAL_MODEL:` line names. If the
|
||||
order is wrong, fix `families` in a reviewed PR.
|
||||
|
||||
<a id="eval-model-generation-unresolved"></a>
|
||||
### `EVAL_MODEL_GENERATION_UNRESOLVED`
|
||||
|
||||
**Meaning.** The rules declare `families`, so every arm must be its family's
|
||||
newest generation. This arm's model is in no family, so its generation cannot be
|
||||
checked, and the arm fails instead of passing unchecked.
|
||||
|
||||
**Fix.** Add the model to its family (newest first) in
|
||||
`.gstack/eval-models.json` in a reviewed PR, or use a model a family lists.
|
||||
@@ -399,6 +399,15 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
(`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete
|
||||
preregistration.
|
||||
|
||||
When the summarizer counts diagnostics, declare them before the freeze commit in a
|
||||
`## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic
|
||||
records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the
|
||||
fixture is built to produce. The summarizer's last stdout line is one JSON object of counts.
|
||||
Then run `prereg --check <path> --allow-repo-commands`. Without the flag, the summarizer is
|
||||
listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`.
|
||||
`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an
|
||||
undeclared one does not match its definition. Fix that before you freeze.
|
||||
|
||||
### Step 1b: Arms and their price
|
||||
|
||||
```bash
|
||||
@@ -412,6 +421,12 @@ and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE
|
||||
means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing
|
||||
page or drop the arm. Copy the priced arms into the Arms section.
|
||||
|
||||
When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families
|
||||
newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and
|
||||
`price-gate` print one `EVAL_MODEL:` line per arm. Pass `--scope smoke` for a smoke run;
|
||||
the default is `full`. A refused arm (`EVAL_MODEL_DENIED`, `EVAL_MODEL_OLD_GENERATION`,
|
||||
`EVAL_MODEL_GENERATION_UNRESOLVED`) is replaced, never argued past.
|
||||
|
||||
## Step 2: zero-cost stub-model dry run
|
||||
|
||||
Run the pipeline over the real development rows with a stub model that costs nothing:
|
||||
|
||||
@@ -71,6 +71,15 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
(`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete
|
||||
preregistration.
|
||||
|
||||
When the summarizer counts diagnostics, declare them before the freeze commit in a
|
||||
`## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic
|
||||
records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the
|
||||
fixture is built to produce. The summarizer's last stdout line is one JSON object of counts.
|
||||
Then run `prereg --check <path> --allow-repo-commands`. Without the flag, the summarizer is
|
||||
listed and the check exits 3. Every `PREREG_DIAGNOSTIC:` line must say `ok`.
|
||||
`PREREG_DIAGNOSTIC_UNTESTED` (exit 1) means a count, a missing diagnostic or an
|
||||
undeclared one does not match its definition. Fix that before you freeze.
|
||||
|
||||
### Step 1b: Arms and their price
|
||||
|
||||
```bash
|
||||
@@ -84,6 +93,12 @@ and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE
|
||||
means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing
|
||||
page or drop the arm. Copy the priced arms into the Arms section.
|
||||
|
||||
When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families
|
||||
newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and
|
||||
`price-gate` print one `EVAL_MODEL:` line per arm. Pass `--scope smoke` for a smoke run;
|
||||
the default is `full`. A refused arm (`EVAL_MODEL_DENIED`, `EVAL_MODEL_OLD_GENERATION`,
|
||||
`EVAL_MODEL_GENERATION_UNRESOLVED`) is replaced, never argued past.
|
||||
|
||||
## Step 2: zero-cost stub-model dry run
|
||||
|
||||
Run the pipeline over the real development rows with a stub model that costs nothing:
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* Field list 2, W9: .gstack/eval-models.json and the lint gstack-eval-plan runs
|
||||
* over every arm (arms, dry-run, pilot start, price-gate). gbrain's rules
|
||||
* ("Fable in smoke only", "no gpt-5.4-mini", "the newest model of each family")
|
||||
* as data; literal allow/deny independent of MODEL_CATALOG; unknown ids fail
|
||||
* closed; the reviewed copy comes from origin/<base>.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { formatEvalCalls } from '../lib/eval-plan';
|
||||
import { lintArmModels, parseEvalModels, type ArmUse, type EvalModelRules } from '../lib/eval-models';
|
||||
import { capyHost, type CapyHost } from './helpers/capy-host';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const GBRAIN_RULES = {
|
||||
schema_version: 1,
|
||||
allow: ['claude-*', 'gpt-*'],
|
||||
deny: ['gpt-5.4-mini'],
|
||||
scopes: { full: { deny: ['claude-fable-*'] } },
|
||||
families: {
|
||||
'claude-opus': ['claude-opus-5-5', 'claude-opus-5', 'claude-opus-4-8'],
|
||||
'claude-fable': ['claude-fable-5-1', 'claude-fable-5'],
|
||||
'gpt-sol': ['gpt-6.1-sol', 'gpt-6-sol'],
|
||||
},
|
||||
};
|
||||
|
||||
function rules(doc: unknown): EvalModelRules {
|
||||
const parsed = parseEvalModels(JSON.stringify(doc));
|
||||
if ('errors' in parsed) throw new Error(parsed.errors.join('; '));
|
||||
return parsed.rules;
|
||||
}
|
||||
const verdicts = (r: EvalModelRules, uses: Array<[string, ArmUse['scope']]>) => lintArmModels(r, uses.map(([model, scope]) => ({ model, scope, where: 't' }))).map(f => `${f.model}/${f.scope}: ${f.verdict} (${f.reason})`);
|
||||
|
||||
describe('lintArmModels', () => {
|
||||
test('gbrain\'s rules: Fable smoke-only, gpt-5.4-mini denied in any form, older generations refused, unknown ids fail closed', () => {
|
||||
expect(verdicts(rules(GBRAIN_RULES), [
|
||||
['claude-fable-5-1', 'smoke'], ['claude-fable-5-1', 'full'], ['gpt-5.4-mini', 'smoke'], ['openai/gpt-5.4-mini', 'full'],
|
||||
['claude-opus-4-8', 'full'], ['anthropic/claude-opus-5-5-20261001', 'full'], ['gpt-6.1-sol', 'full'], ['gemini-3.8-flash', 'smoke'], ['claude-haiku-5-5', 'full'],
|
||||
])).toEqual([
|
||||
'claude-fable-5-1/smoke: ok (allow claude-*; newest in families.claude-fable)',
|
||||
'claude-fable-5-1/full: denied (scopes.full.deny claude-fable-*)',
|
||||
'gpt-5.4-mini/smoke: denied (deny gpt-5.4-mini)',
|
||||
'openai/gpt-5.4-mini/full: denied (deny gpt-5.4-mini)',
|
||||
'claude-opus-4-8/full: old-generation (families.claude-opus newest is claude-opus-5-5)',
|
||||
'anthropic/claude-opus-5-5-20261001/full: ok (allow claude-*; newest in families.claude-opus)',
|
||||
'gpt-6.1-sol/full: ok (allow gpt-*; newest in families.gpt-sol)',
|
||||
'gemini-3.8-flash/smoke: denied (matches no allow entry (unknown ids fail closed))',
|
||||
'claude-haiku-5-5/full: generation-unresolved (in no families entry, so the newest-generation rule cannot be checked)',
|
||||
]);
|
||||
});
|
||||
|
||||
test('literal allow/deny ignore catalog membership: uncatalogued ids are allowed or denied by the file alone; scope allow lists narrow', () => {
|
||||
const r = rules({ schema_version: 1, allow: ['voyage-4-large', 'claude-sonnet-5-5', 'claude-opus-5-5'], deny: ['claude-opus-5-5'], scopes: { smoke: { allow: ['claude-sonnet-*'] } } });
|
||||
expect(verdicts(r, [['voyage-4-large', 'full'], ['claude-opus-5-5', 'full'], ['claude-sonnet-5-5', 'smoke'], ['voyage-4-large', 'smoke']])).toEqual([
|
||||
'voyage-4-large/full: ok (allow voyage-4-large)',
|
||||
'claude-opus-5-5/full: denied (deny claude-opus-5-5)',
|
||||
'claude-sonnet-5-5/smoke: ok (allow claude-sonnet-5-5)',
|
||||
'voyage-4-large/smoke: denied (matches no scopes.smoke.allow entry)',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseEvalModels', () => {
|
||||
test('the file fails closed on a missing allow, unknown keys and scopes, globs in families, duplicates and a malformed id', () => {
|
||||
const errors = (doc: unknown) => { const p = parseEvalModels(JSON.stringify(doc)); return 'errors' in p ? p.errors : []; };
|
||||
expect(errors({ schema_version: 1 })).toEqual(['allow must be a non-empty array of model ids or globs']);
|
||||
expect(errors({ schema_version: 2, allow: ['*'], newest: true })).toEqual(['unknown key "newest" (known: schema_version, allow, deny, scopes, families)', 'schema_version must be 1']);
|
||||
expect(errors({ schema_version: 1, allow: ['*'], scopes: { release: { deny: [] }, smoke: { only: [] } } })).toEqual(['scopes.release: unknown eval scope (known: smoke, full)', 'scopes.smoke: unknown key(s) only (known: allow, deny)']);
|
||||
expect(errors({ schema_version: 1, allow: ['*'], families: { a: ['claude-*'], b: ['x-1', 'x-2'], c: ['x-2'] } })).toEqual([
|
||||
'families.a[0] "claude-*" is not a model id (family entries are literal ids, newest first)', 'families: x-2 is listed in both b and c',
|
||||
]);
|
||||
expect(errors({ schema_version: 1, allow: ['claude-[a-z]+'] })[0]).toContain('allow[0] "claude-[a-z]+" is not a model id or glob');
|
||||
expect(parseEvalModels('{ allow: [] }')).toMatchObject({ errors: [expect.stringContaining('not valid JSON')] });
|
||||
});
|
||||
|
||||
test('MODEL_CATALOG checks only currency: a family listing gstack\'s current smart model after an older id is stale', () => {
|
||||
const p = parseEvalModels(JSON.stringify({ schema_version: 1, allow: ['*'], families: { 'claude-opus': ['claude-opus-5', 'claude-opus-5-5'] } }));
|
||||
expect('errors' in p && p.errors).toEqual([expect.stringMatching(/^families\.claude-opus lists claude-opus-5 before claude-opus-5-5, gstack's current anthropic smart model \(lib\/model-catalog\.ts, verified \d{4}-\d{2}-\d{2}\); list newest first$/)]);
|
||||
expect('rules' in parseEvalModels(JSON.stringify(GBRAIN_RULES))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)', () => {
|
||||
let host: CapyHost;
|
||||
let repo: string;
|
||||
const git = (...args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
};
|
||||
const run = (args: string[]) => {
|
||||
const r = spawnSync(BIN, args, { cwd: repo, env: host.env, encoding: 'utf8', timeout: 60_000 });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
function seed(doc: unknown | null) {
|
||||
const origin = path.join(host.base, 'origin.git');
|
||||
spawnSync('git', ['init', '-q', '--bare', '-b', 'main', origin], { env: host.env, timeout: 30_000 });
|
||||
repo = path.join(host.base, 'repo');
|
||||
fs.mkdirSync(path.join(repo, '.gstack'), { recursive: true });
|
||||
git('init', '-q', '-b', 'main');
|
||||
git('remote', 'add', 'origin', origin);
|
||||
fs.writeFileSync(path.join(repo, 'rows.jsonl'), JSON.stringify({ prompt: 'q1' }) + '\n');
|
||||
fs.writeFileSync(path.join(repo, 'retry.txt'), 'Answer again in the required JSON shape.\n');
|
||||
if (doc) fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify(doc, null, 2) + '\n');
|
||||
git('add', '-A'); git('commit', '-q', '-m', 'seed'); git('push', '-q', 'origin', 'main');
|
||||
}
|
||||
beforeEach(() => { host = capyHost(); });
|
||||
afterEach(() => host.cleanup());
|
||||
|
||||
test('arms: the reviewed rules from origin/main refuse a denied arm; a working-tree edit does not loosen them unless --models-from head', () => {
|
||||
seed(GBRAIN_RULES);
|
||||
const r = run(['arms', '--arms', 'claude-opus-5-5,gpt-5.4-mini']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=flag:--arms');
|
||||
expect(r.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full denied (deny gpt-5.4-mini) where=flag:--arms');
|
||||
expect(r.out).toMatch(/EVAL_MODELS: fail refused=1\/2 source=origin\/main@[0-9a-f]{12}:\.gstack\/eval-models\.json/);
|
||||
expect(r.err).toContain('(EVAL_MODEL_DENIED)');
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ ...GBRAIN_RULES, deny: [] }));
|
||||
const edited = run(['arms', '--arms', 'gpt-5.4-mini']);
|
||||
expect(edited.code).toBe(1);
|
||||
expect(edited.out).toContain('EVAL_MODELS_NOTE: the working-tree .gstack/eval-models.json differs from the reviewed copy; linting with origin/main');
|
||||
const head = run(['arms', '--arms', 'gpt-5.4-mini', '--models-from', 'head']);
|
||||
expect(head.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full generation-unresolved');
|
||||
expect(head.out).toContain('source=head (unreviewed) .gstack/eval-models.json');
|
||||
expect(head.err).toContain('(EVAL_MODEL_GENERATION_UNRESOLVED)');
|
||||
});
|
||||
|
||||
test('dry-run and pilot start: Fable fails the full scope (no reservation), passes --scope smoke; the dry-run record carries the check', () => {
|
||||
seed(GBRAIN_RULES);
|
||||
const out = path.join(host.base, 'out');
|
||||
const base = ['dry-run', '--input', 'rows.jsonl', '--arms', 'claude-fable-5-1', '--retry-prompt', 'retry.txt', '--out', out];
|
||||
const full = run(base);
|
||||
expect(full.code).toBe(1);
|
||||
expect(full.out).toContain('CHECK: model-rules refused claude-fable-5-1 (scope full, dry-run --arms): scopes.full.deny claude-fable-*');
|
||||
expect(full.out).toContain('DRY_RUN_RESULT: fail');
|
||||
const smoke = run([...base, '--scope', 'smoke']);
|
||||
expect(smoke.code).toBe(0);
|
||||
expect(smoke.out).toMatch(/CHECK: model-rules ok \(source origin\/main@[0-9a-f]{12}:\.gstack\/eval-models\.json\)/);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(out, 'dry-run.json'), 'utf8')).checks.model_rules).toMatchObject({ ok: true, refused: [] });
|
||||
const pilotOut = path.join(host.base, 'pilot');
|
||||
const pilot = run(['pilot', 'start', '--out', pilotOut, '--cap', '5', '--arms', 'claude-fable-5-1', '--dry-run', path.join(out, 'dry-run.json')]);
|
||||
expect(pilot.code).toBe(1);
|
||||
expect(pilot.err).toContain('(EVAL_MODEL_DENIED)');
|
||||
expect(fs.existsSync(path.join(pilotOut, 'spend.json'))).toBe(false);
|
||||
expect(run(['pilot', 'start', '--out', pilotOut, '--cap', '5', '--arms', 'claude-fable-5-1', '--dry-run', path.join(out, 'dry-run.json'), '--scope', 'smoke']).code).toBe(0);
|
||||
expect(run([...base, '--scope', 'release']).code).toBe(2);
|
||||
});
|
||||
|
||||
test('price-gate lints each call row at its own scope and writes no receipt on a refusal', () => {
|
||||
seed(GBRAIN_RULES);
|
||||
const head = git('rev-parse', 'HEAD');
|
||||
const gate = 'bun run eval:smoke';
|
||||
const plan = path.join(host.base, 'plan.calls');
|
||||
const rows = [
|
||||
{ purpose: 'smoke', model: 'claude-fable-5-1', calls: 3, in_tokens: 3_000, out_tokens: 1_000, max_out_tokens: 8_192, scope: 'smoke' as const },
|
||||
{ purpose: 'judge', model: 'claude-opus-4-8', calls: 3, in_tokens: 3_000, out_tokens: 1_000, max_out_tokens: 2_048, scope: 'full' as const },
|
||||
];
|
||||
fs.writeFileSync(plan, formatEvalCalls({ gate, head, inputs: [], preconditions: [], rows }));
|
||||
const receipts = path.join(host.base, 'receipts');
|
||||
const r = run(['price-gate', '--gate', gate, '--plan-file', plan, '--out', receipts]);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('EVAL_MODEL: claude-fable-5-1 scope=smoke ok');
|
||||
expect(r.out).toContain('EVAL_MODEL: claude-opus-4-8 scope=full old-generation (families.claude-opus newest is claude-opus-5-5) where=purpose=judge');
|
||||
expect(r.err).toContain('(EVAL_MODEL_OLD_GENERATION)');
|
||||
expect(fs.existsSync(receipts)).toBe(false);
|
||||
fs.writeFileSync(plan, formatEvalCalls({ gate, head, inputs: [], preconditions: [], rows: [rows[0]!] }));
|
||||
const ok = run(['price-gate', '--gate', gate, '--plan-file', plan, '--out', receipts]);
|
||||
expect(ok.code).toBe(0);
|
||||
expect(fs.readdirSync(receipts).length).toBe(1);
|
||||
});
|
||||
|
||||
test('no rules file changes nothing; an invalid reviewed file fails EVAL_MODELS_INVALID for every arm', () => {
|
||||
seed(null);
|
||||
const none = run(['arms', '--arms', 'gpt-5.4-mini']);
|
||||
expect(none.code).toBe(0);
|
||||
expect(none.out).toMatch(/EVAL_MODELS: none at origin\/main@[0-9a-f]{12}\n/);
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ schema_version: 1, allow: ['claude-*'], deny: 'gpt-5.4-mini' }));
|
||||
git('add', '-A'); git('commit', '-q', '-m', 'rules'); git('push', '-q', 'origin', 'main');
|
||||
const invalid = run(['arms', '--arms', 'claude-opus-5-5']);
|
||||
expect(invalid.code).toBe(1);
|
||||
expect(invalid.out).toMatch(/EVAL_MODELS: invalid source=origin\/main@/);
|
||||
expect(invalid.err).toContain('deny must be an array of model ids or globs');
|
||||
expect(invalid.err).toContain('(EVAL_MODELS_INVALID)');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Field list 2, W8: `gstack-eval-plan prereg --check` runs a preregistration's
|
||||
* declared summarizer on its declared synthetic fixture (only with
|
||||
* --allow-repo-commands) and fails PREREG_DIAGNOSTIC_UNTESTED when a count
|
||||
* differs from its preregistered definition, before the freeze commit. The D7
|
||||
* shape (gbrain-evals #162): two diagnostic counts that did not match.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval-plan';
|
||||
import { capyHost, type CapyHost } from './helpers/capy-host';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const SUMMARIZER = [
|
||||
'touch ran.marker',
|
||||
'r=$(grep -c \'"kind":"refusal"\' "$1")',
|
||||
't=$(grep -c \'"kind":"truncated"\' "$1")',
|
||||
'echo "summarizing $1"',
|
||||
'printf \'{"refusals":%s,"truncated":%s}\\n\' "$r" "$t"',
|
||||
].join('\n') + '\n';
|
||||
const RECORDS = ['refusal', 'refusal', 'truncated', 'answer'].map(kind => JSON.stringify({ kind })).join('\n') + '\n';
|
||||
|
||||
function diagnostics(rows: Array<[string, string]>, o: { summarizer?: string; fixture?: string } = {}): string {
|
||||
return `\n## Diagnostics\nsummarizer: ${o.summarizer ?? '`sh summarize.sh {fixture}`'}\nfixture: ${o.fixture ?? 'fixtures/synthetic records.jsonl'}\n\n| diagnostic | expected |\n|---|---|\n${rows.map(([n, c]) => `| ${n} | ${c} |`).join('\n')}\n`;
|
||||
}
|
||||
|
||||
let host: CapyHost;
|
||||
let repo: string;
|
||||
beforeEach(() => {
|
||||
host = capyHost();
|
||||
repo = path.join(host.base, 'repo');
|
||||
fs.mkdirSync(path.join(repo, 'fixtures'), { recursive: true });
|
||||
const init = spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: repo, env: host.env, encoding: 'utf8', timeout: 30_000 });
|
||||
if (init.status !== 0) throw new Error(init.stderr);
|
||||
fs.writeFileSync(path.join(repo, 'summarize.sh'), SUMMARIZER);
|
||||
fs.writeFileSync(path.join(repo, 'fixtures', 'synthetic records.jsonl'), RECORDS);
|
||||
fs.mkdirSync(path.join(repo, 'evals'));
|
||||
});
|
||||
afterEach(() => host.cleanup());
|
||||
|
||||
function check(text: string, ...flags: string[]) {
|
||||
const file = path.join(repo, 'evals', 'prereg.md');
|
||||
fs.writeFileSync(file, text);
|
||||
const r = spawnSync(BIN, ['prereg', '--check', file, ...flags], { cwd: host.base, env: host.env, encoding: 'utf8', timeout: 60_000 });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '', ran: fs.existsSync(path.join(repo, 'ran.marker')) };
|
||||
}
|
||||
|
||||
describe('prereg --check: the Diagnostics section', () => {
|
||||
test('the declared summarizer is listed, not run, without --allow-repo-commands (exit 3)', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]));
|
||||
expect(r.code).toBe(3);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: ok present=5/5 diagnostics=2 file=');
|
||||
expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`);
|
||||
expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)');
|
||||
expect(r.out).not.toContain('PREREG_DIAGNOSTICS:');
|
||||
});
|
||||
|
||||
test('with --allow-repo-commands, matching counts pass and bind the fixture by sha256; the path reaches the command quoted', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]), '--allow-repo-commands');
|
||||
expect(r.err).toBe('');
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.ran).toBe(true);
|
||||
const sha = createHash('sha256').update(RECORDS).digest('hex').slice(0, 12);
|
||||
expect(r.out).toContain('PREREG_DIAGNOSTIC: refusals expected=2 got=2 ok\nPREREG_DIAGNOSTIC: truncated expected=1 got=1 ok\n');
|
||||
expect(r.out).toContain(`PREREG_DIAGNOSTICS: ok n=2 fixture=fixtures/synthetic records.jsonl sha256=${sha}\n`);
|
||||
});
|
||||
|
||||
test('the D7 shape: two counts off their definitions, a declared count the summarizer never reports and an undeclared one all fail PREREG_DIAGNOSTIC_UNTESTED', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', '3'], ['truncated', '0'], ['empty_answers', '0']]), '--allow-repo-commands');
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('PREREG_DIAGNOSTIC: refusals expected=3 got=2 mismatch');
|
||||
expect(r.out).toContain('PREREG_DIAGNOSTIC: truncated expected=0 got=1 mismatch');
|
||||
expect(r.out).toContain('PREREG_DIAGNOSTIC: empty_answers expected=0 got=none missing');
|
||||
expect(r.out).toContain('PREREG_DIAGNOSTICS: untested n=3');
|
||||
expect(r.err).toContain('refusals mismatch (expected 3, got 2); truncated mismatch (expected 0, got 1); empty_answers missing (expected 0, got none)');
|
||||
expect(r.err.trim()).toEndWith('(PREREG_DIAGNOSTIC_UNTESTED)');
|
||||
const undeclared = check(FILLED + diagnostics([['refusals', '2']]), '--allow-repo-commands');
|
||||
expect(undeclared.code).toBe(1);
|
||||
expect(undeclared.out).toContain('PREREG_DIAGNOSTIC: truncated expected=undeclared got=1 undeclared');
|
||||
expect(undeclared.err).toContain('(PREREG_DIAGNOSTIC_UNTESTED)');
|
||||
});
|
||||
|
||||
test('a failing summarizer, non-JSON output and a missing fixture are untested, never passed', () => {
|
||||
fs.writeFileSync(path.join(repo, 'fail.sh'), 'echo "boom: no such column" >&2\nexit 2\n');
|
||||
const failed = check(FILLED + diagnostics([['refusals', '2']], { summarizer: 'sh fail.sh {fixture}' }), '--allow-repo-commands');
|
||||
expect(failed.code).toBe(1);
|
||||
expect(failed.out).toContain('PREREG_DIAGNOSTICS: untested n=0');
|
||||
expect(failed.err).toContain('summarizer exited 2: boom: no such column');
|
||||
const prose = check(FILLED + diagnostics([['refusals', '2']], { summarizer: 'echo refusals=2 {fixture}' }), '--allow-repo-commands');
|
||||
expect(prose.code).toBe(1);
|
||||
expect(prose.err).toContain('not a JSON object of diagnostic counts');
|
||||
const missing = check(FILLED + diagnostics([['refusals', '2']], { fixture: 'fixtures/nope.jsonl' }), '--allow-repo-commands');
|
||||
expect(missing.code).toBe(1);
|
||||
expect(missing.ran).toBe(false);
|
||||
expect(missing.err).toContain('declared fixture fixtures/nope.jsonl is not a file');
|
||||
});
|
||||
|
||||
test('an incomplete declaration is PREREG_INCOMPLETE and never runs, even with --allow-repo-commands', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands');
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: incomplete present=5/5 diagnostics=invalid');
|
||||
expect(r.err).toContain('diagnostics: row "| refusals | two |"');
|
||||
expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})');
|
||||
expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows');
|
||||
expect(r.err.trim()).toEndWith('(PREREG_INCOMPLETE)');
|
||||
});
|
||||
|
||||
test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => {
|
||||
const r = check(FILLED);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toBe(`PREREG: ok present=5/5 file=${path.join(repo, 'evals', 'prereg.md')}\n`);
|
||||
expect(parsePreregDiagnostics(FILLED)).toBeNull();
|
||||
expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n<!-- none -->\n')).toBeNull();
|
||||
expect(preregTemplate()).toContain('## Diagnostics');
|
||||
expect(checkPrereg(preregTemplate()).diagnostics).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user