diff --git a/CHANGELOG.md b/CHANGELOG.md index 2977c693d..b1fded999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.91.373.0] - 2026-10-31 + +- **No-gstack arm in every head-to-head eval.** `/eval-plan` preregistrations now say in a required `## Comparison` section whether any arm loads gstack or another skill stack (`stack: loaded`) or none does (`stack: none` with a `why:` line). A loaded comparison must add a `## No-gstack arm` section and one `#no-gstack` arm per model. That arm uses the same model and inputs, a plain prompt with no skill text, and the same panel and judge, and its result is reported beside the variants. `gstack-eval-plan prereg --check` refuses a loaded comparison without it (`PREREG_NO_GSTACK_ARM`). Arms are now identities `[#]`, priced by the model, so the pilot reserves a variant and its no-gstack arm on the same model separately; a repeated identity is refused (`EVAL_ARM_DUPLICATE`). The rule is written up in `docs/TESTING_INTERNALS.md`. + ## [1.91.372.0] - 2026-10-31 **A test that fails once in CI and passes when run again no longer turns the job red, and every such flake is recorded for the daily flaky-fix pass.** `gstack-flaky run` wraps any test command that writes a JUnit report (Playwright, `bun test`, jest, vitest, pytest). When a few tests fail, it runs the command once more. A test that failed and then passes is a flaky pass, so the job stays green, an Actions `::warning` and a step-summary row name it, and one line goes on a ledger. A rerun counts only when the runner itself reports a full green run, so a real failure, a coverage threshold, a test the first run never reached, a crash or a cancelled job stays red. Before this, each one-off flake meant rerunning the whole CI job by hand (five times in Truther's last evidence window, three of them for one test). diff --git a/VERSION b/VERSION index 4c1d8c4ff..3ffacc691 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.372.0 +1.91.373.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index fe7b8182c..f5ca186bd 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.372.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.373.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index dd65a98f8..36caac274 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -194,7 +194,6 @@ gstack_result_anchor() { ISSUE_UNREADABLE) printf '%s' 'issue-unreadable' ;; ISSUE_LOOKUP_FAILED) printf '%s' 'issue-lookup-failed' ;; ISSUE_LINKS_MISSING) printf '%s' 'issue-links-missing' ;; - PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;; EVAL_PRICE_MISSING) printf '%s' 'eval-price-missing' ;; EVAL_DRY_RUN_FAILED) printf '%s' 'eval-dry-run-failed' ;; EVAL_ARMS_UNRESOLVED) printf '%s' 'eval-arms-unresolved' ;; @@ -240,7 +239,6 @@ gstack_result_anchor() { OUTSIDE_VOICE_UNAVAILABLE) printf '%s' 'outside-voice-unavailable' ;; ARTIFACT_MACHINE_PATH) printf '%s' 'artifact-machine-path' ;; ARTIFACT_PUBLIC_TARGET_EXISTS) printf '%s' 'artifact-public-target-exists' ;; - PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; EVAL_MODELS_INVALID) printf '%s' 'eval-models-invalid' ;; EVAL_MODEL_DENIED) printf '%s' 'eval-model-denied' ;; EVAL_MODEL_OLD_GENERATION) printf '%s' 'eval-model-old-generation' ;; @@ -346,6 +344,10 @@ gstack_result_anchor() { BUDGET_LEDGER_INVALID) printf '%s' 'budget-ledger-invalid' ;; BUDGET_CHECK_SKIPPED) printf '%s' 'budget-check-skipped' ;; HANDOFF_PIN_MOVED) printf '%s' 'handoff-pin-moved' ;; + PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;; + PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; + PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;; + EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;; *) return 1 ;; esac } diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 20eb4b474..1dc081a76 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -6,7 +6,7 @@ * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] - * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] + * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] @@ -31,7 +31,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; +import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; @@ -50,7 +50,7 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json] stub model over the real rows; checklist: price table, output budget, retry prompt text, and with --cap reservation-fits-cap (one call: largest row + output budget, priced) - pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3] + pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] @@ -90,7 +90,7 @@ function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScop if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head'); const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 }); const repoRoot = top.status === 0 ? top.stdout.trim() : null; - const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); + const check = checkModelRules(repoRoot, uses.map(u => (armModel(u.model) === u.model ? u : { ...u, model: armModel(u.model), where: `${u.where} (${u.model})` })), { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n'); for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n'); return check; @@ -116,11 +116,13 @@ function doPrereg(rest: string[]): number { const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null; if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n'); else { - process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); + process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${PREREG_SECTIONS.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`); if (run) process.stdout.write(renderPreregDiagnostics(run)); } - if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n'); + const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)]; + if (incomplete.length) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', incomplete.join('; ')) + '\n'); + if (!r.no_gstack.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_NO_GSTACK_ARM', `no-gstack arm: ${r.no_gstack.problems.join('; ')}`) + '\n'); if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n'); if (!r.ok || (run && !run.ok)) return EXIT.fail; if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; } diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index ecd162ccc..2ed948c98 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1644,6 +1644,51 @@ lower-case words in path components under `test/fixtures/`; a substring match separate change with its own touchfile updates; the list is here so that change can be scoped. +## No-gstack arm + +Garry, 2026-10-31: "all of our head to head tests have to test no gstack from +here so that that case is tested obviously Do it for science". A variant that +beats another variant can still lose to the plain model, and only a no-gstack +arm shows that. + +**Scope.** Every preregistered comparison where any arm loads gstack or another +skill stack: review variants, preamble loadouts, skill diets scored by behavior +evals, and model panels run under gstack or under another stack. The +preregistration says which kind it is in `## Comparison`: `stack: loaded`, or +`stack: none` with a `why:` line (a scorer calibration, for example, where no +arm loads any skill text). `gstack-eval-plan prereg --check` refuses a loaded +comparison without the arm (`PREREG_NO_GSTACK_ARM`). + +**How the arm is built.** One arm per model in the comparison, named +`#no-gstack` in the Arms section and listed on the `arms:` line of +`## No-gstack arm`. Arm identities are `[#]` and are priced by +the model, so the pilot reserves the no-gstack arm beside its variants under its +own attempts. The arm gets: + +- the same model, settings, inputs, fixture and tools as the variants; +- a plain prompt: the task text only, with no SKILL.md, preamble, hooks, + installed skills or routing lines from gstack or any other stack; +- the same panel: the same `E2E_KINDS` kind and trial count, the same judge and + rubric, and the same blinding labels (the judge reads it as `A` or `B` like + any other arm, per [Candidate-side blinding](#candidate-side-blinding)); +- the same budget treatment: it is priced and reserved in the pilot like every + other arm. + +**Reporting.** The no-gstack result is one row beside the variants in every +results table and every verdict, including when it wins. A comparison that +drops the row has not reported its result. The prereg check cannot see the +report, so the `report:` line states where the row goes and the reviewer of +the result holds the author to it. + +**Precedent and its gaps.** `test/helpers/arm-benchmark-harness.ts` already has +a `without-skill` arm beside `with-skill`, and it is the closest existing +control. It falls short of this rule in three ways: each arm and task runs +once rather than as a panel; the judge (`armJudge` in +`test/helpers/llm-judge.ts`) scores each diff on its own, without `A`/`B` +labels; and the candidate's temporary folder is named after the arm +(`arm---*`). Existing paid cases are not retrofitted by the +change that added this rule; an audit of them is tracked separately. + ## Running evals as an agent: detach Moved verbatim from CLAUDE.md (#2096 size limit). The short rule stays in diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c33b6a56a..5f1fb6c28 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3644,14 +3644,44 @@ parentheses; grep the code. ### `PREREG_INCOMPLETE` **Meaning.** `gstack-eval-plan prereg --check ` found a required -preregistration section missing or empty: `## Bars`, `## Arms`, `## Decoys`, -`## Stop rules` or `## Held-out exposure` (every held-out set the plan -touches, its prior exposure and the reservation on it). +preregistration section missing or empty: `## Bars`, `## Arms`, +`## Comparison`, `## Decoys`, `## Stop rules` or `## Held-out exposure` +(every held-out set the plan touches, its prior exposure and the reservation +on it). It also fires when `## Comparison` has no `stack:` line, a value other +than `loaded` or `none`, or `stack: none` without a `why:` line, and when a +`## Diagnostics` declaration is malformed. **Fix.** Fill the named sections before any priced call. A plan that cannot name its stop rule or its held-out exposure is not preregistered; the pilot refuses to start without a passing check. + +### `PREREG_NO_GSTACK_ARM` + +**Meaning.** The preregistration says `stack: loaded` (some arm loads gstack or +another skill stack) and its `## No-gstack arm` section is missing, lacks one +of `arms:`, `inputs:`, `prompt:`, `panel:` or `report:`, lists an arm that +is not a `#no-gstack` Arms line, or leaves a model in the comparison +without its own no-gstack arm. Arms match on whole identities, so +`claude-opus-5#no-gstack` does not cover `claude-opus-5-5`. + +**Fix.** Add one `#no-gstack` line per model under `## Arms`, list them +on the `arms:` line, and fill the other four keys: the same inputs, a plain +prompt with no skill text, the same panel and judge, and where the result row +goes beside the variants. When no arm loads a skill stack, write +`stack: none` with a `why:` line instead. The rule is in +[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm). + + +### `EVAL_ARM_DUPLICATE` + +**Meaning.** `gstack-eval-plan pilot start --arms` named the same arm identity +twice, so two arms would share one set of pilot attempts. + +**Fix.** Give each arm its own identity: `` or `#`, for +example `claude-opus-5-5` and `claude-opus-5-5#no-gstack`. Nothing was +reserved. + ### `EVAL_PRICE_MISSING` diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 9200ee065..5ec40e511 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -135,10 +135,12 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| | Bars | one line per metric: name, pass bar, direction, what a miss means for the decision | -| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b | +| Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | +| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | +| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init @@ -149,6 +151,16 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack +(review variants, preamble loadouts, skill diets, a model panel under a stack) also +runs each model with no skill stack at all. Add `#no-gstack` to Arms for every +model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows +and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed +skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels; +`report:` its row beside the variants in every results table and verdict. +`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows +`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out. + When the summarizer counts diagnostics, declare them before the freeze commit in a `## Diagnostics` section: `summarizer: `, `fixture: `, and one `| diagnostic | expected |` row per diagnostic, with the count the @@ -169,7 +181,9 @@ falling back to the repo's own policy document (`--policy-doc `), or from `--arms ` when the user named them. Each `ARM:` line carries its source and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`) means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing -page or drop the arm. Copy the priced arms into the Arms section. +page or drop the arm. Copy the priced arms into the Arms section. An arm identity +`#` (for example `claude-opus-5-5#no-gstack`) is priced by its model and +gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`. When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 39ad672ee..019499eca 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -57,10 +57,12 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| | Bars | one line per metric: name, pass bar, direction, what a miss means for the decision | -| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b | +| Arms | one line per arm: the arm identity first (`` or `#`), settings, per-item cost from Step 1b | +| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | +| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init @@ -71,6 +73,16 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack +(review variants, preamble loadouts, skill diets, a model panel under a stack) also +runs each model with no skill stack at all. Add `#no-gstack` to Arms for every +model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows +and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed +skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels; +`report:` its row beside the variants in every results table and verdict. +`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows +`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out. + When the summarizer counts diagnostics, declare them before the freeze commit in a `## Diagnostics` section: `summarizer: `, `fixture: `, and one `| diagnostic | expected |` row per diagnostic, with the count the @@ -91,7 +103,9 @@ falling back to the repo's own policy document (`--policy-doc `), or from `--arms ` when the user named them. Each `ARM:` line carries its source and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`) means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing -page or drop the arm. Copy the priced arms into the Arms section. +page or drop the arm. Copy the priced arms into the Arms section. An arm identity +`#` (for example `claude-opus-5-5#no-gstack`) is priced by its model and +gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`. When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 33542c5fa..6bb74db94 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -23,8 +23,14 @@ import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type // --------------------------------------------------------------------------- // preregistration // --------------------------------------------------------------------------- -export const PREREG_SECTIONS = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; -export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; diagnostics: PreregDiagnostics | null } +export const PREREG_SECTIONS = ['Bars', 'Arms', 'Comparison', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; +/** `## Comparison`: `stack: loaded` when any arm loads gstack or another skill stack, `stack: none` with a `why:` otherwise. */ +export interface ComparisonCheck { stack: 'loaded' | 'none' | null; why: string | null; errors: string[] } +/** `## No-gstack arm`, required when `stack: loaded`: one `#no-gstack` Arms line per model, and the inputs, prompt, panel and report it shares with the variants. */ +export interface NoGstackArmCheck { required: boolean; ok: boolean; arms: string[]; problems: string[] } +export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null } +export const NO_GSTACK_LOADOUT = 'no-gstack'; +export const NO_GSTACK_KEYS = ['arms', 'inputs', 'prompt', 'panel', 'report'] as const; export function preregTemplate(title = 'Eval preregistration'): string { return `# ${title} @@ -33,7 +39,13 @@ export function preregTemplate(title = 'Eval preregistration'): string { ## Arms - + + +## Comparison + + +## No-gstack arm + ## Decoys @@ -47,14 +59,71 @@ export function preregTemplate(title = 'Eval preregistration'): string { `; } -/** A section's content lines (blank lines and one-line `` guidance dropped); null when the heading is absent. */ +/** A section's content lines (blank lines and `` guidance, one-line or multi-line, dropped); null when the heading is absent. */ function sectionBody(text: string, name: string): string[] | null { const lines = text.split('\n'); const headings = lines.map((l, i) => ({ i, m: /^#{1,6}\s+(.+?)\s*$/.exec(l) })).filter(x => x.m) as Array<{ i: number; m: RegExpExecArray }>; const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, '')); if (idx < 0) return null; const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length; - return lines.slice(headings[idx]!.i + 1, end).filter(l => l.trim() && !/^\s*\s*$/.test(l)); + return lines.slice(headings[idx]!.i + 1, end).join('\n').replace(//g, '').split('\n').filter(l => l.trim()); +} + +/** The model an arm identity `[#]` is priced by. */ +export const armModel = (id: string) => id.split('#')[0]!; +const armLoadout = (id: string) => (id.includes('#') ? id.slice(id.indexOf('#') + 1) : null); + +/** The first token of each Arms line (bullets, table pipes and backticks stripped; a table's header row skipped). */ +export function armIdentities(text: string): string[] { + const body = sectionBody(text, 'Arms') ?? []; + const row = (l: string) => /^\s*\|/.test(l); + const separator = (l: string | undefined) => !!l && /^[\s:|-]+$/.test(l) && l.includes('-'); + const ids: string[] = []; + body.forEach((line, i) => { + if (separator(line) || (row(line) && separator(body[i + 1]))) return; + const token = line.replace(/^\s*(?:[-*+]|\d+[.)])?\s*\|?\s*/, '').split(/[\s|]+/)[0]!.replace(/[`,;:]+$/g, '').replace(/^`/, ''); + if (token) ids.push(token); + }); + return ids; +} + +const keyValues = (body: string[]) => { + const kv: Record = {}; + for (const line of body) { + const m = /^\s*(?:[-*]\s+)?([a-z]+):(.*)$/i.exec(line); + if (m) kv[m[1]!.toLowerCase()] = m[2]!.trim(); + } + return kv; +}; + +function checkComparison(text: string): ComparisonCheck { + const body = sectionBody(text, 'Comparison'); + if (!body?.length) return { stack: null, why: null, errors: [] }; + const kv = keyValues(body); + const why = kv.why || null; + if (kv.stack === undefined) return { stack: null, why, errors: ['no stack: line (stack: loaded or stack: none)'] }; + if (kv.stack !== 'loaded' && kv.stack !== 'none') return { stack: null, why, errors: [`stack: ${kv.stack} is neither loaded nor none`] }; + return { stack: kv.stack, why, errors: kv.stack === 'none' && !why ? ['stack: none needs a why: line saying why no arm loads a skill stack'] : [] }; +} + +function checkNoGstackArm(text: string, stack: ComparisonCheck['stack']): NoGstackArmCheck { + if (stack !== 'loaded') return { required: false, ok: true, arms: [], problems: [] }; + const body = sectionBody(text, 'No-gstack arm'); + if (!body?.length) return { required: true, ok: false, arms: [], problems: ['missing section'] }; + const kv = keyValues(body); + const problems = NO_GSTACK_KEYS.filter(k => !kv[k]).map(k => `missing ${k}`); + const listed = (kv.arms ?? '').split(/[\s,]+/).map(a => a.replace(/^`|`$/g, '')).filter(Boolean); + const ids = armIdentities(text); + for (const a of listed) { + if (armLoadout(a) !== NO_GSTACK_LOADOUT) problems.push(`${a} is not a #${NO_GSTACK_LOADOUT} identity`); + else if (!ids.includes(a)) problems.push(`${a} is not an Arms line`); + } + const loaded = [...new Set(ids.filter(id => armLoadout(id) !== NO_GSTACK_LOADOUT).map(armModel))]; + for (const m of loaded) { + const arm = `${m}#${NO_GSTACK_LOADOUT}`; + if (!ids.includes(arm) || !listed.includes(arm)) problems.push(`no no-gstack arm for ${m}`); + } + return { required: true, ok: problems.length === 0, arms: listed, problems }; } export function checkPrereg(text: string): PreregCheck { @@ -63,8 +132,10 @@ export function checkPrereg(text: string): PreregCheck { const body = sectionBody(text, name); if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name); } + const comparison = checkComparison(text); + const no_gstack = checkNoGstackArm(text, comparison.stack); const diagnostics = parsePreregDiagnostics(text); - return { ok: missing.length === 0 && empty.length === 0 && !diagnostics?.errors.length, present, missing, empty, diagnostics }; + return { ok: missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length, present, missing, empty, comparison, no_gstack, diagnostics }; } // --------------------------------------------------------------------------- @@ -152,8 +223,8 @@ export const DEFAULT_TOKENS: TokenEstimate = { input: 4_000, output: 2_000 }; const POLICY_DOCS = ['docs/model-policy.md', 'MODEL_POLICY.md', 'docs/MODELS.md']; function priceArm(model: string, source: string, provider: string | null, tokens: TokenEstimate): Arm { - const row = pricingRow(model); - return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, model) : null }; + const row = pricingRow(armModel(model)); + return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, armModel(model)) : null }; } /** Models named in the repo's policy document that the price table knows. */ @@ -249,10 +320,10 @@ export function dryRun(o: DryRunOptions): DryRunResult { if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t }); }); const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null; - const missingPrice = o.arms.filter(a => !hasPricing(a)); + const missingPrice = o.arms.filter(a => !hasPricing(armModel(a))); const perItem: Record = {}; const avgIn = rows ? Math.ceil(inputTokens / rows) : 0; - for (const a of o.arms) perItem[a] = hasPricing(a) ? estimateCostUsd({ input: avgIn, output: budget }, a) : null; + for (const a of o.arms) perItem[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: avgIn, output: budget }, armModel(a)) : null; const checks = { price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice }, output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR }, @@ -260,7 +331,7 @@ export function dryRun(o: DryRunOptions): DryRunResult { } as DryRunResult['checks']; if (o.cap !== undefined) { const perArm: Record = {}; - for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null; + for (const a of o.arms) perArm[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: maxInput, output: budget }, armModel(a)) : null; checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm }; } if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) }; @@ -303,15 +374,19 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult; if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } }; const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget }; + const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; + const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`; + const keys = o.arms.map(keyOf); + const duplicates = [...new Set(o.arms.filter((_, i) => keys.indexOf(keys[i]!) !== i).map(a => o.arms.filter(b => keyOf(b) === keyOf(a)).join(' = ')))]; + if (duplicates.length) return { record, error: { code: 'EVAL_ARM_DUPLICATE', message: `arm ${duplicates.join(', ')} is listed more than once or shares one ledger attempt name; give each arm its own # identity` } }; fs.mkdirSync(o.out, { recursive: true }); const ledgerFile = path.join(o.out, PILOT_FILES.spend); - const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; let error: { code: ResultCodeName; message: string } | undefined; outer: for (const model of o.arms) { - if (!hasPricing(model)) { error = { code: 'EVAL_PRICE_MISSING', message: `${model} has no row in lib/pricing.ts` }; break; } + if (!hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; } for (let i = 1; i <= o.n; i++) { - const estimate = estimateCostUsd(tokens, model); - const attempt = `pilot-${model.replace(/[^A-Za-z0-9.-]/g, '_')}-${i}`; + const estimate = estimateCostUsd(tokens, armModel(model)); + const attempt = `pilot-${keyOf(model)}-${i}`; try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}`, now: o.now }); } catch (e) { if (e instanceof SpendCapExceeded) { record.status = 'budget_exhausted'; error = { code: 'EVAL_BUDGET_EXHAUSTED', message: e.message }; break outer; } diff --git a/lib/result-codes-eval.ts b/lib/result-codes-eval.ts new file mode 100644 index 000000000..70102382b --- /dev/null +++ b/lib/result-codes-eval.ts @@ -0,0 +1,33 @@ +/** + * Result codes for the /eval-plan preregistration (bin/gstack-eval-plan prereg + * --check, the priced pilot's arm list). Kept beside lib/result-codes.ts, which + * spreads this table into RESULT_CODES; the same rules apply: one stable + * docs/troubleshooting.md anchor, a summary and a fix clause each. The + * no-gstack arm rule is documented in docs/TESTING_INTERNALS.md#no-gstack-arm. + */ +import type { ResultCode } from './result-codes'; + +export const EVAL_RESULT_CODES = { + PREREG_INCOMPLETE: { + anchor: 'prereg-incomplete', + summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison or Diagnostics declaration is malformed', + fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, comparison, decoys, stop rules, held-out exposure) before any priced call', + }, + // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. + PREREG_DIAGNOSTIC_UNTESTED: { + anchor: 'prereg-diagnostic-untested', + summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count', + fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted', + }, + // Garry, 2026-10-31: "all of our head to head tests have to test no gstack from here". + PREREG_NO_GSTACK_ARM: { + anchor: 'prereg-no-gstack-arm', + summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own #no-gstack Arms line', + fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each #no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line', + }, + EVAL_ARM_DUPLICATE: { + anchor: 'eval-arm-duplicate', + summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts', + fix: 'give each arm its own identity, or # (for example claude-opus-5-5#no-gstack beside claude-opus-5-5); nothing was reserved', + }, +} as const satisfies Record; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index 082924de1..beaeb779b 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -13,6 +13,7 @@ import { RELEASE_RESULT_CODES } from './result-codes-release'; import { CAPY_RESULT_CODES } from './result-codes-capy'; import { BUDGET_RESULT_CODES } from './result-codes-budget'; +import { EVAL_RESULT_CODES } from './result-codes-eval'; export interface ResultCode { /** Stable `` in docs/troubleshooting.md. */ @@ -435,11 +436,6 @@ export const RESULT_CODES = { }, // PR E (multi-agent wave): gstack-eval-plan, gstack-lane-check, gstack-evidence verify, // gstack-owner-brief, gstack-contributor-mode, gstack-banned-terms. - PREREG_INCOMPLETE: { - anchor: 'prereg-incomplete', - summary: 'the preregistration file lacks a required section or a section is empty', - fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, decoys, stop rules, held-out exposure) before any priced call', - }, EVAL_PRICE_MISSING: { anchor: 'eval-price-missing', summary: 'an arm names a model with no row in the price table, so its admission cannot be estimated', @@ -671,12 +667,6 @@ export const RESULT_CODES = { summary: '`gstack-autoplan export --public ` writes only into an empty directory or over an earlier public copy of the same run', fix: 'pass an empty or new --public directory, or the directory that already holds this run\'s public copy', }, - // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. - PREREG_DIAGNOSTIC_UNTESTED: { - anchor: 'prereg-diagnostic-untested', - summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count', - fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted', - }, // Field list 2 (2026-10-16), W9: .gstack/eval-models.json. EVAL_MODELS_INVALID: { anchor: 'eval-models-invalid', @@ -772,6 +762,7 @@ export const RESULT_CODES = { ...RELEASE_RESULT_CODES, ...CAPY_RESULT_CODES, ...BUDGET_RESULT_CODES, + ...EVAL_RESULT_CODES, } as const satisfies Record; export type ResultCodeName = keyof typeof RESULT_CODES; diff --git a/package.json b/package.json index 3884072da..540545d72 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.372", + "version": "1.91.373", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index b0a107407..e40ae2c35 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ // The build entry list: every compiled binary, which skills reach through their `--outfile` path instead. 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. - 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', + 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). 'test/helpers/exercised-commands.json', ]; diff --git a/test/eval-models.test.ts b/test/eval-models.test.ts index 6e22a4e03..e907b7e6e 100644 --- a/test/eval-models.test.ts +++ b/test/eval-models.test.ts @@ -142,6 +142,19 @@ describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)', expect(head.err).toContain('(EVAL_MODEL_GENERATION_UNRESOLVED)'); }); + test('a # arm is linted as its model: a deny still refuses it and an exact allow accepts it', () => { + seed(GBRAIN_RULES); + const r = run(['arms', '--arms', 'claude-opus-5-5#no-gstack,gpt-5.4-mini#no-gstack']); + expect(r.code).toBe(1); + expect(r.out).toContain('EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=flag:--arms (claude-opus-5-5#no-gstack)'); + expect(r.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full denied (deny gpt-5.4-mini) where=flag:--arms (gpt-5.4-mini#no-gstack)'); + expect(r.err).toContain('(EVAL_MODEL_DENIED)'); + fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ schema_version: 1, allow: ['claude-opus-5-5'], deny: [] })); + const exact = run(['arms', '--arms', 'claude-opus-5-5,claude-opus-5-5#no-gstack', '--models-from', 'head']); + expect(exact.code).toBe(0); + expect(exact.out).toContain('where=flag:--arms (claude-opus-5-5#no-gstack)'); + }); + test('dry-run and pilot start: Fable fails the full scope (no reservation), passes --scope smoke; the dry-run record carries the check', () => { seed(GBRAIN_RULES); const out = path.join(host.base, 'out'); diff --git a/test/eval-plan-no-gstack-arm.test.ts b/test/eval-plan-no-gstack-arm.test.ts new file mode 100644 index 000000000..c04e4de9b --- /dev/null +++ b/test/eval-plan-no-gstack-arm.test.ts @@ -0,0 +1,199 @@ +/** + * The no-gstack arm (Garry, 2026-10-31: "all of our head to head tests have to + * test no gstack from here"). A preregistration says in `## Comparison` whether + * any arm loads gstack or another skill stack (`stack: loaded`) or none does + * (`stack: none` with a `why:`). A loaded comparison needs a `## No-gstack arm` + * section naming one `#no-gstack` arm per model, each also an Arms line, + * plus the inputs, prompt, panel and report it shares with the variants. Arms are + * identities `[#]`, priced by the model, so the no-gstack arm on + * the same model gets its own pilot reservations. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { armModel, checkPrereg, dryRun, preregTemplate, resolveArms, startPilot } from '../lib/eval-plan'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const dirs: string[] = []; +const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; }; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); + +const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`; +const LOADED = '\n## Comparison\nstack: loaded\n'; +const NO_GSTACK = (o: Partial> = {}) => { + const v = { arms: 'claude-opus-5-5#no-gstack', inputs: 'the same rows and fixture as the variants', prompt: 'the task text only; no SKILL.md, preamble, hooks or installed skills', panel: 'behavior, 3 trials, the same judge and labels', report: 'one row beside the variants in the results table', ...o }; + return `\n## No-gstack arm\n${Object.entries(v).filter(([, x]) => x !== undefined).map(([k, x]) => `${k}: ${x}`).join('\n')}\n`; +}; +const VARIANTS = ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#review-h', 'claude-opus-5-5#no-gstack'); + +describe('the Comparison declaration', () => { + test('Comparison is a required section; the blank template fails and multi-line guidance never counts as content', () => { + const t = preregTemplate('x'); + expect(t).toContain('## Comparison'); + expect(t).toContain('## No-gstack arm'); + const blank = checkPrereg(t); + expect(blank.empty).toContain('Comparison'); + expect(checkPrereg(BASE + VARIANTS + '\n## Comparison\n\n').empty).toContain('Comparison'); + }); + + test('stack: none needs a why: line and passes without the arm; an unknown value fails', () => { + const none = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration; no arm loads a skill stack\n'); + expect(none.ok).toBe(true); + expect(none.comparison.stack).toBe('none'); + expect(none.no_gstack.required).toBe(false); + const noWhy = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\n'); + expect(noWhy.ok).toBe(false); + expect(noWhy.comparison.errors.join(';')).toContain('why:'); + const odd = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: maybe\n'); + expect(odd.ok).toBe(false); + expect(odd.comparison.errors.join(';')).toContain('stack: maybe'); + }); +}); + +describe('the No-gstack arm section', () => { + test('a loaded comparison passes with a complete section and fails without it', () => { + expect(checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK()).ok).toBe(true); + const missing = checkPrereg(BASE + VARIANTS + LOADED); + expect(missing.ok).toBe(false); + expect(missing.no_gstack).toMatchObject({ required: true, ok: false }); + expect(missing.no_gstack.problems).toEqual(['missing section']); + }); + + for (const key of ['arms', 'inputs', 'prompt', 'panel', 'report'] as const) { + test(`a missing or blank ${key}: line fails naming it`, () => { + for (const v of [undefined, ' ']) { + const r = checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK({ [key]: v } as Record)); + expect(r.ok).toBe(false); + expect(r.no_gstack.problems).toContain(`missing ${key}`); + } + }); + } + + test('every model that loads a stack needs its own #no-gstack arm: a two-model panel needs two', () => { + const panel = ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack'); + const one = checkPrereg(BASE + panel + LOADED + NO_GSTACK()); + expect(one.ok).toBe(false); + expect(one.no_gstack.problems).toEqual(['no no-gstack arm for gpt-6.1-sol']); + const both = checkPrereg(BASE + ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack', 'gpt-6.1-sol#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack, gpt-6.1-sol#no-gstack' })); + expect(both.ok).toBe(true); + }); + + test('a model panel under another skill stack (not gstack) is loaded too and needs the same arms', () => { + const other = ARMS('claude-opus-5-5#other-stack', 'gpt-6.1-sol#other-stack'); + const r = checkPrereg(BASE + other + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack' })); + expect(r.ok).toBe(false); + expect(r.no_gstack.problems).toEqual(['claude-opus-5-5#no-gstack is not an Arms line', 'no no-gstack arm for claude-opus-5-5', 'no no-gstack arm for gpt-6.1-sol']); + }); + + test('a listed arm must be an Arms line and a #no-gstack identity, matched on whole identities', () => { + const notListed = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g') + LOADED + NO_GSTACK()); + expect(notListed.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line'); + const prefix = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5#no-gstack' })); + expect(prefix.no_gstack.problems).toContain('no no-gstack arm for claude-opus-5-5'); + const wrongLoadout = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#plain') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#plain' })); + expect(wrongLoadout.no_gstack.problems).toContain('claude-opus-5-5#plain is not a #no-gstack identity'); + const prose = checkPrereg(BASE + '\n## Arms\n- claude-opus-5-5#review-g, compared with claude-opus-5-5#no-gstack in prose\n' + LOADED + NO_GSTACK()); + expect(prose.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line'); + }); + + test('a table in Arms: the header row is not an arm', () => { + const table = '\n## Arms\n| model | settings |\n|---|---|\n| claude-opus-5-5#review-g | default |\n| claude-opus-5-5#no-gstack | default |\n'; + expect(checkPrereg(BASE + table + LOADED + NO_GSTACK()).ok).toBe(true); + }); +}); + +describe('arm identities in pricing and the pilot', () => { + test('armModel strips the loadout; arms and the dry run price #no-gstack by its model', () => { + expect(armModel('anthropic/claude-opus-5-5#no-gstack')).toBe('anthropic/claude-opus-5-5'); + expect(armModel('gpt-6.1-sol')).toBe('gpt-6.1-sol'); + const d = tmp(); + const arms = resolveArms({ cwd: d, explicit: ['claude-opus-5-5#no-gstack'] }); + expect(arms.errors).toEqual([]); + expect(arms.arms[0]).toMatchObject({ model: 'claude-opus-5-5#no-gstack', priced: true }); + const rows = path.join(d, 'rows.jsonl'); + fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n'); + const dry = dryRun({ input: rows, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'] }); + expect(dry.checks.price_table.ok).toBe(true); + expect(dry.per_item_usd['claude-opus-5-5#no-gstack']).toBe(dry.per_item_usd['claude-opus-5-5']); + }); + + const passingDryRun = (d: string) => { + const rows = path.join(d, 'rows.jsonl'); + fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n'); + const file = path.join(d, 'dry-run.json'); + const r = dryRun({ input: rows, arms: ['claude-opus-5-5'], retryPrompt: (fs.writeFileSync(path.join(d, 'retry.md'), 'try again\n'), path.join(d, 'retry.md')) }); + fs.writeFileSync(file, JSON.stringify(r)); + expect(r.pass, JSON.stringify(r.checks)).toBe(true); + return file; + }; + + test('the pilot reserves a variant and its no-gstack arm on the same model under distinct attempts; the total is the sum', () => { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 2, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'], dryRun: passingDryRun(d) }); + expect(r.error).toBeUndefined(); + const ids = r.record.attempts.map(a => a.attempt); + expect(new Set(ids).size).toBe(4); + const ledger = JSON.parse(fs.readFileSync(path.join(d, 'pilot', fs.readdirSync(path.join(d, 'pilot')).find(f => f.includes('spend'))!), 'utf8')); + const reserved = JSON.stringify(ledger); + for (const id of ids) expect(reserved).toContain(id); + const each = r.record.attempts[0]!.estimate_usd; + expect(r.record.attempts.every(a => a.estimate_usd === each)).toBe(true); + }); + + test('a duplicate arm identity is refused before anything is reserved (EVAL_ARM_DUPLICATE)', () => { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms: ['claude-opus-5-5', 'claude-opus-5-5'], dryRun: passingDryRun(d) }); + expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE'); + expect(r.record.attempts).toEqual([]); + }); + + test('distinct identities that share one ledger attempt name are refused before anything is reserved', () => { + for (const arms of [['claude-opus-5-5#review/a', 'claude-opus-5-5#review_a'], ['gpt-6#1-sol', 'gpt-6.1-sol']]) { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms, dryRun: passingDryRun(d) }); + expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE'); + expect(r.error?.message).toContain(arms.join(' = ')); + expect(r.record.attempts).toEqual([]); + expect(fs.existsSync(path.join(d, 'pilot'))).toBe(false); + } + }); +}); + +describe('gstack-eval-plan prereg --check', () => { + const run = (text: string) => { + const d = tmp(); + const file = path.join(d, 'prereg.md'); + fs.writeFileSync(file, text); + const r = spawnSync(BIN, ['prereg', '--check', file], { cwd: d, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; + }; + + test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => { + const r = run(BASE + VARIANTS + LOADED); + expect(r.code).toBe(1); + expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m); + expect(r.err).toContain('missing section'); + expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)'); + expect(r.err).not.toContain('(PREREG_INCOMPLETE)'); + }); + + test('missing sections and arm problems both print, PREREG_INCOMPLETE first', () => { + const r = run(VARIANTS + LOADED); + expect(r.code).toBe(1); + const a = r.err.indexOf('(PREREG_INCOMPLETE)'), b = r.err.indexOf('(PREREG_NO_GSTACK_ARM)'); + expect(a).toBeGreaterThanOrEqual(0); + expect(b).toBeGreaterThan(a); + }); + + test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => { + const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK()); + expect(ok.code, ok.out + ok.err).toBe(0); + expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m); + const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n'); + expect(none.code, none.out + none.err).toBe(0); + expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m); + }); +}); diff --git a/test/eval-plan-prereg-diagnostics.test.ts b/test/eval-plan-prereg-diagnostics.test.ts index 7271cc7a8..9e9e18a20 100644 --- a/test/eval-plan-prereg-diagnostics.test.ts +++ b/test/eval-plan-prereg-diagnostics.test.ts @@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval import { capyHost, type CapyHost } from './helpers/capy-host'; const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); -const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n'; const SUMMARIZER = [ 'touch ran.marker', 'r=$(grep -c \'"kind":"refusal"\' "$1")', @@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']])); expect(r.code).toBe(3); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: ok present=5/5 diagnostics=2 file='); + expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file='); expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`); expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)'); expect(r.out).not.toContain('PREREG_DIAGNOSTICS:'); @@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands'); expect(r.code).toBe(1); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: incomplete present=5/5 diagnostics=invalid'); + expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid'); expect(r.err).toContain('diagnostics: row "| refusals | two |"'); expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})'); expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows'); @@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => { test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => { const r = check(FILLED); expect(r.code).toBe(0); - expect(r.out).toBe(`PREREG: ok present=5/5 file=${path.join(repo, 'evals', 'prereg.md')}\n`); + expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`); expect(parsePreregDiagnostics(FILLED)).toBeNull(); expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n\n')).toBeNull(); expect(preregTemplate()).toContain('## Diagnostics'); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index a91653450..d56450722 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -101,6 +101,7 @@ { "file": "lib/result-codes-release.ts", "movedFrom": "lib/result-codes.ts (the release-path rows: ship queue, pregate timing, approval classes, state matrix; field list 2 wave B)" }, { "file": "lib/result-codes-capy.ts", "movedFrom": "lib/result-codes.ts (the Capy rows: approval record, Captain brief, PR screenshots; GSTA-61)" }, { "file": "lib/result-codes-budget.ts", "movedFrom": "lib/result-codes.ts (the /ship token budget and handoff rows; GSTA-152)" }, + { "file": "lib/result-codes-eval.ts", "movedFrom": "lib/result-codes.ts (the /eval-plan preregistration rows and the no-gstack arm; GSTA-168)" }, { "file": "lib/ship-budget.ts", "movedFrom": "(new: /ship token budget ledger, meters and admission, GSTA-152)" }, { "file": "lib/ship-budget-cli.ts", "movedFrom": "(new: bin/gstack-ship-budget start/check/settle/close/resume, GSTA-152)" }, { "file": "lib/ship-handoff.ts", "movedFrom": "(new: /ship fresh-context handoff, verify, return and reconcile, GSTA-152)" }, diff --git a/test/gstack-doctor.test.ts b/test/gstack-doctor.test.ts index d39ab1440..1becf6c50 100644 --- a/test/gstack-doctor.test.ts +++ b/test/gstack-doctor.test.ts @@ -27,7 +27,7 @@ const COPIED = [ 'autoplan/SKILL.md', // its frontmatter registers the autoplan hook the hooks-row test breaks 'bin/gstack-launch-probe.sh', 'bin/gstack-doctor-check.sh', 'bin/gstack-doctor-components.sh', 'bin/gstack-runtime-pins.ts', 'lib/runtime-pins.ts', - 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', // the windows bash row + 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', // the windows bash row ]; const bases: string[] = []; afterEach(() => { for (const b of bases.splice(0)) fs.rmSync(b, { recursive: true, force: true }); }); diff --git a/test/plan-tune-cathedral.test.ts b/test/plan-tune-cathedral.test.ts index e0c73fcf3..20109dc42 100644 --- a/test/plan-tune-cathedral.test.ts +++ b/test/plan-tune-cathedral.test.ts @@ -70,7 +70,7 @@ function scaffoldFixture(workDir: string): { workDir: string; stateRoot: string; // These imports are resolved relative to the copied install, not the source checkout. const libDir = path.join(workDir, 'lib'); fs.mkdirSync(libDir, { recursive: true }); - for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts']) { + for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts', 'result-codes-eval.ts']) { fs.copyFileSync(path.join(ROOT, 'lib', file), path.join(libDir, file)); } diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index 661ed5742..8784d88a0 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -50,7 +50,8 @@ describe('preregistration', () => { expect(check.missing).toEqual([]); expect(check.empty.length).toBe(PREREG_SECTIONS.length); expect(check.ok).toBe(false); - const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in'); + expect(PREREG_SECTIONS).toContain('Comparison'); + const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack'); expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] }); expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]); }); diff --git a/test/timeline-stop-hook-slug.test.ts b/test/timeline-stop-hook-slug.test.ts index 0c9975788..857df2304 100644 --- a/test/timeline-stop-hook-slug.test.ts +++ b/test/timeline-stop-hook-slug.test.ts @@ -33,6 +33,7 @@ const HOOK_FILES = [ 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', + 'lib/result-codes-eval.ts', ]; const SLUG = 'stop-hook-slug-test'; const BUDGET_MS = 2000;