mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
v1.91.373.0 feat(eval-plan): require a no-gstack arm in every head-to-head preregistration (#3690)
* feat(eval-plan): require a no-gstack arm in every head-to-head preregistration Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(eval-plan): lint loadout arms by their model; refuse arms sharing a ledger attempt Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
44eeeb3c9b
commit
0cd2b4a9f9
22 files changed
+477
-52
No files matched your search
@@ -1,5 +1,9 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.373.0] - 2026-10-31
|
||||
|
||||
- **No-gstack arm in every head-to-head eval.** `/eval-plan` preregistrations now say in a required `## Comparison` section whether any arm loads gstack or another skill stack (`stack: loaded`) or none does (`stack: none` with a `why:` line). A loaded comparison must add a `## No-gstack arm` section and one `<model>#no-gstack` arm per model. That arm uses the same model and inputs, a plain prompt with no skill text, and the same panel and judge, and its result is reported beside the variants. `gstack-eval-plan prereg --check` refuses a loaded comparison without it (`PREREG_NO_GSTACK_ARM`). Arms are now identities `<model>[#<loadout>]`, priced by the model, so the pilot reserves a variant and its no-gstack arm on the same model separately; a repeated identity is refused (`EVAL_ARM_DUPLICATE`). The rule is written up in `docs/TESTING_INTERNALS.md`.
|
||||
|
||||
## [1.91.372.0] - 2026-10-31
|
||||
|
||||
**A test that fails once in CI and passes when run again no longer turns the job red, and every such flake is recorded for the daily flaky-fix pass.** `gstack-flaky run` wraps any test command that writes a JUnit report (Playwright, `bun test`, jest, vitest, pytest). When a few tests fail, it runs the command once more. A test that failed and then passes is a flaky pass, so the job stays green, an Actions `::warning` and a step-summary row name it, and one line goes on a ledger. A rerun counts only when the runner itself reports a full green run, so a real failure, a coverage threshold, a test the first run never reached, a crash or a cancelled job stays red. Before this, each one-off flake meant rerunning the whole CI job by hand (five times in Truther's last evidence window, three of them for one test).
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.372.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.373.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
@@ -194,7 +194,6 @@ gstack_result_anchor() {
|
||||
ISSUE_UNREADABLE) printf '%s' 'issue-unreadable' ;;
|
||||
ISSUE_LOOKUP_FAILED) printf '%s' 'issue-lookup-failed' ;;
|
||||
ISSUE_LINKS_MISSING) printf '%s' 'issue-links-missing' ;;
|
||||
PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;;
|
||||
EVAL_PRICE_MISSING) printf '%s' 'eval-price-missing' ;;
|
||||
EVAL_DRY_RUN_FAILED) printf '%s' 'eval-dry-run-failed' ;;
|
||||
EVAL_ARMS_UNRESOLVED) printf '%s' 'eval-arms-unresolved' ;;
|
||||
@@ -240,7 +239,6 @@ gstack_result_anchor() {
|
||||
OUTSIDE_VOICE_UNAVAILABLE) printf '%s' 'outside-voice-unavailable' ;;
|
||||
ARTIFACT_MACHINE_PATH) printf '%s' 'artifact-machine-path' ;;
|
||||
ARTIFACT_PUBLIC_TARGET_EXISTS) printf '%s' 'artifact-public-target-exists' ;;
|
||||
PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;;
|
||||
EVAL_MODELS_INVALID) printf '%s' 'eval-models-invalid' ;;
|
||||
EVAL_MODEL_DENIED) printf '%s' 'eval-model-denied' ;;
|
||||
EVAL_MODEL_OLD_GENERATION) printf '%s' 'eval-model-old-generation' ;;
|
||||
@@ -346,6 +344,10 @@ gstack_result_anchor() {
|
||||
BUDGET_LEDGER_INVALID) printf '%s' 'budget-ledger-invalid' ;;
|
||||
BUDGET_CHECK_SKIPPED) printf '%s' 'budget-check-skipped' ;;
|
||||
HANDOFF_PIN_MOVED) printf '%s' 'handoff-pin-moved' ;;
|
||||
PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;;
|
||||
PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;;
|
||||
PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;;
|
||||
EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
* gstack-eval-plan arms [--arms <model,model>] [--policy-doc <file>] [--input-tokens N] [--output-tokens N] [--json]
|
||||
* gstack-eval-plan dry-run --input <rows.jsonl> --arms <model,model> [--fields a,b] [--output-budget N]
|
||||
* [--retry-prompt <file>] [--max-rows N] [--out <dir>] [--json]
|
||||
* gstack-eval-plan pilot start --out <dir> --cap <usd> --arms <model,model> --dry-run <dry-run.json> [--n 3]
|
||||
* gstack-eval-plan pilot start --out <dir> --cap <usd> --arms <model[#loadout],...> --dry-run <dry-run.json> [--n 3]
|
||||
* gstack-eval-plan pilot settle --out <dir> --attempt <id> --usd <n|unknown>
|
||||
* gstack-eval-plan pilot release --out <dir> --attempt <id>
|
||||
* gstack-eval-plan pilot report --out <dir> --total <items> [--workers W] [--json]
|
||||
@@ -31,7 +31,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
@@ -50,7 +50,7 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
|
||||
dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json]
|
||||
stub model over the real rows; checklist: price table, output budget, retry prompt text,
|
||||
and with --cap reservation-fits-cap (one call: largest row + output budget, priced)
|
||||
pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3]
|
||||
pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3]
|
||||
pilot settle --out dir --attempt id --usd <n|unknown> pilot release --out dir --attempt id
|
||||
pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask
|
||||
price-gate --gate-file <file> | --gate "<exact gate command>" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out dir] [--json]
|
||||
@@ -90,7 +90,7 @@ function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScop
|
||||
if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head');
|
||||
const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 });
|
||||
const repoRoot = top.status === 0 ? top.stdout.trim() : null;
|
||||
const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' });
|
||||
const check = checkModelRules(repoRoot, uses.map(u => (armModel(u.model) === u.model ? u : { ...u, model: armModel(u.model), where: `${u.where} (${u.model})` })), { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' });
|
||||
if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n');
|
||||
for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n');
|
||||
return check;
|
||||
@@ -116,11 +116,13 @@ function doPrereg(rest: string[]): number {
|
||||
const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null;
|
||||
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n');
|
||||
else {
|
||||
process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`);
|
||||
process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${PREREG_SECTIONS.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`);
|
||||
if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`);
|
||||
if (run) process.stdout.write(renderPreregDiagnostics(run));
|
||||
}
|
||||
if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n');
|
||||
const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)];
|
||||
if (incomplete.length) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', incomplete.join('; ')) + '\n');
|
||||
if (!r.no_gstack.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_NO_GSTACK_ARM', `no-gstack arm: ${r.no_gstack.problems.join('; ')}`) + '\n');
|
||||
if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n');
|
||||
if (!r.ok || (run && !run.ok)) return EXIT.fail;
|
||||
if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; }
|
||||
|
||||
@@ -1644,6 +1644,51 @@ lower-case words in path components under `test/fixtures/`; a substring match
|
||||
separate change with its own touchfile updates; the list is here so that change
|
||||
can be scoped.
|
||||
|
||||
## No-gstack arm
|
||||
|
||||
Garry, 2026-10-31: "all of our head to head tests have to test no gstack from
|
||||
here so that that case is tested obviously Do it for science". A variant that
|
||||
beats another variant can still lose to the plain model, and only a no-gstack
|
||||
arm shows that.
|
||||
|
||||
**Scope.** Every preregistered comparison where any arm loads gstack or another
|
||||
skill stack: review variants, preamble loadouts, skill diets scored by behavior
|
||||
evals, and model panels run under gstack or under another stack. The
|
||||
preregistration says which kind it is in `## Comparison`: `stack: loaded`, or
|
||||
`stack: none` with a `why:` line (a scorer calibration, for example, where no
|
||||
arm loads any skill text). `gstack-eval-plan prereg --check` refuses a loaded
|
||||
comparison without the arm (`PREREG_NO_GSTACK_ARM`).
|
||||
|
||||
**How the arm is built.** One arm per model in the comparison, named
|
||||
`<model>#no-gstack` in the Arms section and listed on the `arms:` line of
|
||||
`## No-gstack arm`. Arm identities are `<model>[#<loadout>]` and are priced by
|
||||
the model, so the pilot reserves the no-gstack arm beside its variants under its
|
||||
own attempts. The arm gets:
|
||||
|
||||
- the same model, settings, inputs, fixture and tools as the variants;
|
||||
- a plain prompt: the task text only, with no SKILL.md, preamble, hooks,
|
||||
installed skills or routing lines from gstack or any other stack;
|
||||
- the same panel: the same `E2E_KINDS` kind and trial count, the same judge and
|
||||
rubric, and the same blinding labels (the judge reads it as `A` or `B` like
|
||||
any other arm, per [Candidate-side blinding](#candidate-side-blinding));
|
||||
- the same budget treatment: it is priced and reserved in the pilot like every
|
||||
other arm.
|
||||
|
||||
**Reporting.** The no-gstack result is one row beside the variants in every
|
||||
results table and every verdict, including when it wins. A comparison that
|
||||
drops the row has not reported its result. The prereg check cannot see the
|
||||
report, so the `report:` line states where the row goes and the reviewer of
|
||||
the result holds the author to it.
|
||||
|
||||
**Precedent and its gaps.** `test/helpers/arm-benchmark-harness.ts` already has
|
||||
a `without-skill` arm beside `with-skill`, and it is the closest existing
|
||||
control. It falls short of this rule in three ways: each arm and task runs
|
||||
once rather than as a panel; the judge (`armJudge` in
|
||||
`test/helpers/llm-judge.ts`) scores each diff on its own, without `A`/`B`
|
||||
labels; and the candidate's temporary folder is named after the arm
|
||||
(`arm-<fixture>-<arm>-*`). Existing paid cases are not retrofitted by the
|
||||
change that added this rule; an audit of them is tracked separately.
|
||||
|
||||
## Running evals as an agent: detach
|
||||
|
||||
Moved verbatim from CLAUDE.md (#2096 size limit). The short rule stays in
|
||||
|
||||
+33
-3
@@ -3644,14 +3644,44 @@ parentheses; grep the code.
|
||||
### `PREREG_INCOMPLETE`
|
||||
|
||||
**Meaning.** `gstack-eval-plan prereg --check <file>` found a required
|
||||
preregistration section missing or empty: `## Bars`, `## Arms`, `## Decoys`,
|
||||
`## Stop rules` or `## Held-out exposure` (every held-out set the plan
|
||||
touches, its prior exposure and the reservation on it).
|
||||
preregistration section missing or empty: `## Bars`, `## Arms`,
|
||||
`## Comparison`, `## Decoys`, `## Stop rules` or `## Held-out exposure`
|
||||
(every held-out set the plan touches, its prior exposure and the reservation
|
||||
on it). It also fires when `## Comparison` has no `stack:` line, a value other
|
||||
than `loaded` or `none`, or `stack: none` without a `why:` line, and when a
|
||||
`## Diagnostics` declaration is malformed.
|
||||
|
||||
**Fix.** Fill the named sections before any priced call. A plan that cannot
|
||||
name its stop rule or its held-out exposure is not preregistered; the pilot
|
||||
refuses to start without a passing check.
|
||||
|
||||
<a id="prereg-no-gstack-arm"></a>
|
||||
### `PREREG_NO_GSTACK_ARM`
|
||||
|
||||
**Meaning.** The preregistration says `stack: loaded` (some arm loads gstack or
|
||||
another skill stack) and its `## No-gstack arm` section is missing, lacks one
|
||||
of `arms:`, `inputs:`, `prompt:`, `panel:` or `report:`, lists an arm that
|
||||
is not a `<model>#no-gstack` Arms line, or leaves a model in the comparison
|
||||
without its own no-gstack arm. Arms match on whole identities, so
|
||||
`claude-opus-5#no-gstack` does not cover `claude-opus-5-5`.
|
||||
|
||||
**Fix.** Add one `<model>#no-gstack` line per model under `## Arms`, list them
|
||||
on the `arms:` line, and fill the other four keys: the same inputs, a plain
|
||||
prompt with no skill text, the same panel and judge, and where the result row
|
||||
goes beside the variants. When no arm loads a skill stack, write
|
||||
`stack: none` with a `why:` line instead. The rule is in
|
||||
[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm).
|
||||
|
||||
<a id="eval-arm-duplicate"></a>
|
||||
### `EVAL_ARM_DUPLICATE`
|
||||
|
||||
**Meaning.** `gstack-eval-plan pilot start --arms` named the same arm identity
|
||||
twice, so two arms would share one set of pilot attempts.
|
||||
|
||||
**Fix.** Give each arm its own identity: `<model>` or `<model>#<loadout>`, for
|
||||
example `claude-opus-5-5` and `claude-opus-5-5#no-gstack`. Nothing was
|
||||
reserved.
|
||||
|
||||
<a id="eval-price-missing"></a>
|
||||
### `EVAL_PRICE_MISSING`
|
||||
|
||||
|
||||
+16
-2
@@ -135,10 +135,12 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
| Section | What it holds |
|
||||
|---|---|
|
||||
| Bars | one line per metric: name, pass bar, direction, what a miss means for the decision |
|
||||
| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b |
|
||||
| Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b |
|
||||
| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
|
||||
| Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
|
||||
| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` |
|
||||
| Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
|
||||
| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) |
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init <path>
|
||||
@@ -149,6 +151,16 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
(`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete
|
||||
preregistration.
|
||||
|
||||
**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack
|
||||
(review variants, preamble loadouts, skill diets, a model panel under a stack) also
|
||||
runs each model with no skill stack at all. Add `<model>#no-gstack` to Arms for every
|
||||
model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows
|
||||
and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed
|
||||
skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels;
|
||||
`report:` its row beside the variants in every results table and verdict.
|
||||
`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows
|
||||
`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out.
|
||||
|
||||
When the summarizer counts diagnostics, declare them before the freeze commit in a
|
||||
`## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic
|
||||
records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the
|
||||
@@ -169,7 +181,9 @@ falling back to the repo's own policy document (`--policy-doc <path>`), or from
|
||||
`--arms <picked-models>` when the user named them. Each `ARM:` line carries its source
|
||||
and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`)
|
||||
means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing
|
||||
page or drop the arm. Copy the priced arms into the Arms section.
|
||||
page or drop the arm. Copy the priced arms into the Arms section. An arm identity
|
||||
`<model>#<loadout>` (for example `claude-opus-5-5#no-gstack`) is priced by its model and
|
||||
gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`.
|
||||
|
||||
When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families
|
||||
newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and
|
||||
|
||||
+16
-2
@@ -57,10 +57,12 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
| Section | What it holds |
|
||||
|---|---|
|
||||
| Bars | one line per metric: name, pass bar, direction, what a miss means for the decision |
|
||||
| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b |
|
||||
| Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b |
|
||||
| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line |
|
||||
| Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves |
|
||||
| Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` |
|
||||
| Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it |
|
||||
| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) |
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init <path>
|
||||
@@ -71,6 +73,16 @@ Write (or open) the preregistration file. Each section is required and must be f
|
||||
(`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete
|
||||
preregistration.
|
||||
|
||||
**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack
|
||||
(review variants, preamble loadouts, skill diets, a model panel under a stack) also
|
||||
runs each model with no skill stack at all. Add `<model>#no-gstack` to Arms for every
|
||||
model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows
|
||||
and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed
|
||||
skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels;
|
||||
`report:` its row beside the variants in every results table and verdict.
|
||||
`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows
|
||||
`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out.
|
||||
|
||||
When the summarizer counts diagnostics, declare them before the freeze commit in a
|
||||
`## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic
|
||||
records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the
|
||||
@@ -91,7 +103,9 @@ falling back to the repo's own policy document (`--policy-doc <path>`), or from
|
||||
`--arms <picked-models>` when the user named them. Each `ARM:` line carries its source
|
||||
and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`)
|
||||
means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing
|
||||
page or drop the arm. Copy the priced arms into the Arms section.
|
||||
page or drop the arm. Copy the priced arms into the Arms section. An arm identity
|
||||
`<model>#<loadout>` (for example `claude-opus-5-5#no-gstack`) is priced by its model and
|
||||
gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`.
|
||||
|
||||
When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families
|
||||
newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and
|
||||
|
||||
+90
-15
@@ -23,8 +23,14 @@ import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type
|
||||
// ---------------------------------------------------------------------------
|
||||
// preregistration
|
||||
// ---------------------------------------------------------------------------
|
||||
export const PREREG_SECTIONS = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'] as const;
|
||||
export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; diagnostics: PreregDiagnostics | null }
|
||||
export const PREREG_SECTIONS = ['Bars', 'Arms', 'Comparison', 'Decoys', 'Stop rules', 'Held-out exposure'] as const;
|
||||
/** `## Comparison`: `stack: loaded` when any arm loads gstack or another skill stack, `stack: none` with a `why:` otherwise. */
|
||||
export interface ComparisonCheck { stack: 'loaded' | 'none' | null; why: string | null; errors: string[] }
|
||||
/** `## No-gstack arm`, required when `stack: loaded`: one `<model>#no-gstack` Arms line per model, and the inputs, prompt, panel and report it shares with the variants. */
|
||||
export interface NoGstackArmCheck { required: boolean; ok: boolean; arms: string[]; problems: string[] }
|
||||
export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null }
|
||||
export const NO_GSTACK_LOADOUT = 'no-gstack';
|
||||
export const NO_GSTACK_KEYS = ['arms', 'inputs', 'prompt', 'panel', 'report'] as const;
|
||||
|
||||
export function preregTemplate(title = 'Eval preregistration'): string {
|
||||
return `# ${title}
|
||||
@@ -33,7 +39,13 @@ export function preregTemplate(title = 'Eval preregistration'): string {
|
||||
<!-- One line per metric: name, the pass bar, the direction, and what a miss means for the decision. -->
|
||||
|
||||
## Arms
|
||||
<!-- One line per arm: model id (must be in lib/pricing.ts), settings, and the per-item cost from \`gstack-eval-plan arms\`. -->
|
||||
<!-- One line per arm: the arm identity first (\`<model>\` or \`<model>#<loadout>\`, the model in lib/pricing.ts, e.g. \`claude-opus-5-5#no-gstack\`), settings, and the per-item cost from \`gstack-eval-plan arms\`. -->
|
||||
|
||||
## Comparison
|
||||
<!-- \`stack: loaded\` when any arm loads gstack or another skill stack (review variants, preamble loadouts, skill diets, a model panel under a stack); otherwise \`stack: none\` and a \`why:\` line. -->
|
||||
|
||||
## No-gstack arm
|
||||
<!-- Required with \`stack: loaded\`: \`arms:\` one \`<model>#no-gstack\` Arms line per model; \`inputs:\` the same rows and fixture; \`prompt:\` the task only (no SKILL.md, preamble, hooks or installed skills); \`panel:\` the same E2E kind, trials and judge; \`report:\` its row beside the variants. -->
|
||||
|
||||
## Decoys
|
||||
<!-- Inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves. -->
|
||||
@@ -47,14 +59,71 @@ export function preregTemplate(title = 'Eval preregistration'): string {
|
||||
`;
|
||||
}
|
||||
|
||||
/** A section's content lines (blank lines and one-line `<!-- -->` guidance dropped); null when the heading is absent. */
|
||||
/** A section's content lines (blank lines and `<!-- -->` guidance, one-line or multi-line, dropped); null when the heading is absent. */
|
||||
function sectionBody(text: string, name: string): string[] | null {
|
||||
const lines = text.split('\n');
|
||||
const headings = lines.map((l, i) => ({ i, m: /^#{1,6}\s+(.+?)\s*$/.exec(l) })).filter(x => x.m) as Array<{ i: number; m: RegExpExecArray }>;
|
||||
const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, ''));
|
||||
if (idx < 0) return null;
|
||||
const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length;
|
||||
return lines.slice(headings[idx]!.i + 1, end).filter(l => l.trim() && !/^\s*<!--.*-->\s*$/.test(l));
|
||||
return lines.slice(headings[idx]!.i + 1, end).join('\n').replace(/<!--[\s\S]*?-->/g, '').split('\n').filter(l => l.trim());
|
||||
}
|
||||
|
||||
/** The model an arm identity `<model>[#<loadout>]` is priced by. */
|
||||
export const armModel = (id: string) => id.split('#')[0]!;
|
||||
const armLoadout = (id: string) => (id.includes('#') ? id.slice(id.indexOf('#') + 1) : null);
|
||||
|
||||
/** The first token of each Arms line (bullets, table pipes and backticks stripped; a table's header row skipped). */
|
||||
export function armIdentities(text: string): string[] {
|
||||
const body = sectionBody(text, 'Arms') ?? [];
|
||||
const row = (l: string) => /^\s*\|/.test(l);
|
||||
const separator = (l: string | undefined) => !!l && /^[\s:|-]+$/.test(l) && l.includes('-');
|
||||
const ids: string[] = [];
|
||||
body.forEach((line, i) => {
|
||||
if (separator(line) || (row(line) && separator(body[i + 1]))) return;
|
||||
const token = line.replace(/^\s*(?:[-*+]|\d+[.)])?\s*\|?\s*/, '').split(/[\s|]+/)[0]!.replace(/[`,;:]+$/g, '').replace(/^`/, '');
|
||||
if (token) ids.push(token);
|
||||
});
|
||||
return ids;
|
||||
}
|
||||
|
||||
const keyValues = (body: string[]) => {
|
||||
const kv: Record<string, string> = {};
|
||||
for (const line of body) {
|
||||
const m = /^\s*(?:[-*]\s+)?([a-z]+):(.*)$/i.exec(line);
|
||||
if (m) kv[m[1]!.toLowerCase()] = m[2]!.trim();
|
||||
}
|
||||
return kv;
|
||||
};
|
||||
|
||||
function checkComparison(text: string): ComparisonCheck {
|
||||
const body = sectionBody(text, 'Comparison');
|
||||
if (!body?.length) return { stack: null, why: null, errors: [] };
|
||||
const kv = keyValues(body);
|
||||
const why = kv.why || null;
|
||||
if (kv.stack === undefined) return { stack: null, why, errors: ['no stack: line (stack: loaded or stack: none)'] };
|
||||
if (kv.stack !== 'loaded' && kv.stack !== 'none') return { stack: null, why, errors: [`stack: ${kv.stack} is neither loaded nor none`] };
|
||||
return { stack: kv.stack, why, errors: kv.stack === 'none' && !why ? ['stack: none needs a why: line saying why no arm loads a skill stack'] : [] };
|
||||
}
|
||||
|
||||
function checkNoGstackArm(text: string, stack: ComparisonCheck['stack']): NoGstackArmCheck {
|
||||
if (stack !== 'loaded') return { required: false, ok: true, arms: [], problems: [] };
|
||||
const body = sectionBody(text, 'No-gstack arm');
|
||||
if (!body?.length) return { required: true, ok: false, arms: [], problems: ['missing section'] };
|
||||
const kv = keyValues(body);
|
||||
const problems = NO_GSTACK_KEYS.filter(k => !kv[k]).map(k => `missing ${k}`);
|
||||
const listed = (kv.arms ?? '').split(/[\s,]+/).map(a => a.replace(/^`|`$/g, '')).filter(Boolean);
|
||||
const ids = armIdentities(text);
|
||||
for (const a of listed) {
|
||||
if (armLoadout(a) !== NO_GSTACK_LOADOUT) problems.push(`${a} is not a <model>#${NO_GSTACK_LOADOUT} identity`);
|
||||
else if (!ids.includes(a)) problems.push(`${a} is not an Arms line`);
|
||||
}
|
||||
const loaded = [...new Set(ids.filter(id => armLoadout(id) !== NO_GSTACK_LOADOUT).map(armModel))];
|
||||
for (const m of loaded) {
|
||||
const arm = `${m}#${NO_GSTACK_LOADOUT}`;
|
||||
if (!ids.includes(arm) || !listed.includes(arm)) problems.push(`no no-gstack arm for ${m}`);
|
||||
}
|
||||
return { required: true, ok: problems.length === 0, arms: listed, problems };
|
||||
}
|
||||
|
||||
export function checkPrereg(text: string): PreregCheck {
|
||||
@@ -63,8 +132,10 @@ export function checkPrereg(text: string): PreregCheck {
|
||||
const body = sectionBody(text, name);
|
||||
if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name);
|
||||
}
|
||||
const comparison = checkComparison(text);
|
||||
const no_gstack = checkNoGstackArm(text, comparison.stack);
|
||||
const diagnostics = parsePreregDiagnostics(text);
|
||||
return { ok: missing.length === 0 && empty.length === 0 && !diagnostics?.errors.length, present, missing, empty, diagnostics };
|
||||
return { ok: missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length, present, missing, empty, comparison, no_gstack, diagnostics };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -152,8 +223,8 @@ export const DEFAULT_TOKENS: TokenEstimate = { input: 4_000, output: 2_000 };
|
||||
const POLICY_DOCS = ['docs/model-policy.md', 'MODEL_POLICY.md', 'docs/MODELS.md'];
|
||||
|
||||
function priceArm(model: string, source: string, provider: string | null, tokens: TokenEstimate): Arm {
|
||||
const row = pricingRow(model);
|
||||
return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, model) : null };
|
||||
const row = pricingRow(armModel(model));
|
||||
return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, armModel(model)) : null };
|
||||
}
|
||||
|
||||
/** Models named in the repo's policy document that the price table knows. */
|
||||
@@ -249,10 +320,10 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t });
|
||||
});
|
||||
const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null;
|
||||
const missingPrice = o.arms.filter(a => !hasPricing(a));
|
||||
const missingPrice = o.arms.filter(a => !hasPricing(armModel(a)));
|
||||
const perItem: Record<string, number | null> = {};
|
||||
const avgIn = rows ? Math.ceil(inputTokens / rows) : 0;
|
||||
for (const a of o.arms) perItem[a] = hasPricing(a) ? estimateCostUsd({ input: avgIn, output: budget }, a) : null;
|
||||
for (const a of o.arms) perItem[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: avgIn, output: budget }, armModel(a)) : null;
|
||||
const checks = {
|
||||
price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice },
|
||||
output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR },
|
||||
@@ -260,7 +331,7 @@ export function dryRun(o: DryRunOptions): DryRunResult {
|
||||
} as DryRunResult['checks'];
|
||||
if (o.cap !== undefined) {
|
||||
const perArm: Record<string, number | null> = {};
|
||||
for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null;
|
||||
for (const a of o.arms) perArm[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: maxInput, output: budget }, armModel(a)) : null;
|
||||
checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm };
|
||||
}
|
||||
if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) };
|
||||
@@ -303,15 +374,19 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?:
|
||||
const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult;
|
||||
if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } };
|
||||
const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget };
|
||||
const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' };
|
||||
const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`;
|
||||
const keys = o.arms.map(keyOf);
|
||||
const duplicates = [...new Set(o.arms.filter((_, i) => keys.indexOf(keys[i]!) !== i).map(a => o.arms.filter(b => keyOf(b) === keyOf(a)).join(' = ')))];
|
||||
if (duplicates.length) return { record, error: { code: 'EVAL_ARM_DUPLICATE', message: `arm ${duplicates.join(', ')} is listed more than once or shares one ledger attempt name; give each arm its own <model>#<loadout> identity` } };
|
||||
fs.mkdirSync(o.out, { recursive: true });
|
||||
const ledgerFile = path.join(o.out, PILOT_FILES.spend);
|
||||
const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' };
|
||||
let error: { code: ResultCodeName; message: string } | undefined;
|
||||
outer: for (const model of o.arms) {
|
||||
if (!hasPricing(model)) { error = { code: 'EVAL_PRICE_MISSING', message: `${model} has no row in lib/pricing.ts` }; break; }
|
||||
if (!hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; }
|
||||
for (let i = 1; i <= o.n; i++) {
|
||||
const estimate = estimateCostUsd(tokens, model);
|
||||
const attempt = `pilot-${model.replace(/[^A-Za-z0-9.-]/g, '_')}-${i}`;
|
||||
const estimate = estimateCostUsd(tokens, armModel(model));
|
||||
const attempt = `pilot-${keyOf(model)}-${i}`;
|
||||
try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}`, now: o.now }); }
|
||||
catch (e) {
|
||||
if (e instanceof SpendCapExceeded) { record.status = 'budget_exhausted'; error = { code: 'EVAL_BUDGET_EXHAUSTED', message: e.message }; break outer; }
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Result codes for the /eval-plan preregistration (bin/gstack-eval-plan prereg
|
||||
* --check, the priced pilot's arm list). Kept beside lib/result-codes.ts, which
|
||||
* spreads this table into RESULT_CODES; the same rules apply: one stable
|
||||
* docs/troubleshooting.md anchor, a summary and a fix clause each. The
|
||||
* no-gstack arm rule is documented in docs/TESTING_INTERNALS.md#no-gstack-arm.
|
||||
*/
|
||||
import type { ResultCode } from './result-codes';
|
||||
|
||||
export const EVAL_RESULT_CODES = {
|
||||
PREREG_INCOMPLETE: {
|
||||
anchor: 'prereg-incomplete',
|
||||
summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison or Diagnostics declaration is malformed',
|
||||
fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, comparison, decoys, stop rules, held-out exposure) before any priced call',
|
||||
},
|
||||
// Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit.
|
||||
PREREG_DIAGNOSTIC_UNTESTED: {
|
||||
anchor: 'prereg-diagnostic-untested',
|
||||
summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count',
|
||||
fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted',
|
||||
},
|
||||
// Garry, 2026-10-31: "all of our head to head tests have to test no gstack from here".
|
||||
PREREG_NO_GSTACK_ARM: {
|
||||
anchor: 'prereg-no-gstack-arm',
|
||||
summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own <model>#no-gstack Arms line',
|
||||
fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each <model>#no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line',
|
||||
},
|
||||
EVAL_ARM_DUPLICATE: {
|
||||
anchor: 'eval-arm-duplicate',
|
||||
summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts',
|
||||
fix: 'give each arm its own identity, <model> or <model>#<loadout> (for example claude-opus-5-5#no-gstack beside claude-opus-5-5); nothing was reserved',
|
||||
},
|
||||
} as const satisfies Record<string, ResultCode>;
|
||||
+2
-11
@@ -13,6 +13,7 @@
|
||||
import { RELEASE_RESULT_CODES } from './result-codes-release';
|
||||
import { CAPY_RESULT_CODES } from './result-codes-capy';
|
||||
import { BUDGET_RESULT_CODES } from './result-codes-budget';
|
||||
import { EVAL_RESULT_CODES } from './result-codes-eval';
|
||||
|
||||
export interface ResultCode {
|
||||
/** Stable `<a id>` in docs/troubleshooting.md. */
|
||||
@@ -435,11 +436,6 @@ export const RESULT_CODES = {
|
||||
},
|
||||
// PR E (multi-agent wave): gstack-eval-plan, gstack-lane-check, gstack-evidence verify,
|
||||
// gstack-owner-brief, gstack-contributor-mode, gstack-banned-terms.
|
||||
PREREG_INCOMPLETE: {
|
||||
anchor: 'prereg-incomplete',
|
||||
summary: 'the preregistration file lacks a required section or a section is empty',
|
||||
fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, decoys, stop rules, held-out exposure) before any priced call',
|
||||
},
|
||||
EVAL_PRICE_MISSING: {
|
||||
anchor: 'eval-price-missing',
|
||||
summary: 'an arm names a model with no row in the price table, so its admission cannot be estimated',
|
||||
@@ -671,12 +667,6 @@ export const RESULT_CODES = {
|
||||
summary: '`gstack-autoplan export --public <dir>` writes only into an empty directory or over an earlier public copy of the same run',
|
||||
fix: 'pass an empty or new --public directory, or the directory that already holds this run\'s public copy',
|
||||
},
|
||||
// Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit.
|
||||
PREREG_DIAGNOSTIC_UNTESTED: {
|
||||
anchor: 'prereg-diagnostic-untested',
|
||||
summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count',
|
||||
fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted',
|
||||
},
|
||||
// Field list 2 (2026-10-16), W9: .gstack/eval-models.json.
|
||||
EVAL_MODELS_INVALID: {
|
||||
anchor: 'eval-models-invalid',
|
||||
@@ -772,6 +762,7 @@ export const RESULT_CODES = {
|
||||
...RELEASE_RESULT_CODES,
|
||||
...CAPY_RESULT_CODES,
|
||||
...BUDGET_RESULT_CODES,
|
||||
...EVAL_RESULT_CODES,
|
||||
} as const satisfies Record<string, ResultCode>;
|
||||
|
||||
export type ResultCodeName = keyof typeof RESULT_CODES;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.372",
|
||||
"version": "1.91.373",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
@@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
|
||||
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
|
||||
'scripts/build.sh',
|
||||
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts',
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts',
|
||||
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
|
||||
'test/helpers/exercised-commands.json',
|
||||
];
|
||||
|
||||
@@ -142,6 +142,19 @@ describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)',
|
||||
expect(head.err).toContain('(EVAL_MODEL_GENERATION_UNRESOLVED)');
|
||||
});
|
||||
|
||||
test('a <model>#<loadout> arm is linted as its model: a deny still refuses it and an exact allow accepts it', () => {
|
||||
seed(GBRAIN_RULES);
|
||||
const r = run(['arms', '--arms', 'claude-opus-5-5#no-gstack,gpt-5.4-mini#no-gstack']);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toContain('EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=flag:--arms (claude-opus-5-5#no-gstack)');
|
||||
expect(r.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full denied (deny gpt-5.4-mini) where=flag:--arms (gpt-5.4-mini#no-gstack)');
|
||||
expect(r.err).toContain('(EVAL_MODEL_DENIED)');
|
||||
fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ schema_version: 1, allow: ['claude-opus-5-5'], deny: [] }));
|
||||
const exact = run(['arms', '--arms', 'claude-opus-5-5,claude-opus-5-5#no-gstack', '--models-from', 'head']);
|
||||
expect(exact.code).toBe(0);
|
||||
expect(exact.out).toContain('where=flag:--arms (claude-opus-5-5#no-gstack)');
|
||||
});
|
||||
|
||||
test('dry-run and pilot start: Fable fails the full scope (no reservation), passes --scope smoke; the dry-run record carries the check', () => {
|
||||
seed(GBRAIN_RULES);
|
||||
const out = path.join(host.base, 'out');
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* The no-gstack arm (Garry, 2026-10-31: "all of our head to head tests have to
|
||||
* test no gstack from here"). A preregistration says in `## Comparison` whether
|
||||
* any arm loads gstack or another skill stack (`stack: loaded`) or none does
|
||||
* (`stack: none` with a `why:`). A loaded comparison needs a `## No-gstack arm`
|
||||
* section naming one `<model>#no-gstack` arm per model, each also an Arms line,
|
||||
* plus the inputs, prompt, panel and report it shares with the variants. Arms are
|
||||
* identities `<model>[#<loadout>]`, priced by the model, so the no-gstack arm on
|
||||
* the same model gets its own pilot reservations.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { armModel, checkPrereg, dryRun, preregTemplate, resolveArms, startPilot } from '../lib/eval-plan';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const dirs: string[] = [];
|
||||
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; };
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`;
|
||||
const LOADED = '\n## Comparison\nstack: loaded\n';
|
||||
const NO_GSTACK = (o: Partial<Record<'arms' | 'inputs' | 'prompt' | 'panel' | 'report', string>> = {}) => {
|
||||
const v = { arms: 'claude-opus-5-5#no-gstack', inputs: 'the same rows and fixture as the variants', prompt: 'the task text only; no SKILL.md, preamble, hooks or installed skills', panel: 'behavior, 3 trials, the same judge and labels', report: 'one row beside the variants in the results table', ...o };
|
||||
return `\n## No-gstack arm\n${Object.entries(v).filter(([, x]) => x !== undefined).map(([k, x]) => `${k}: ${x}`).join('\n')}\n`;
|
||||
};
|
||||
const VARIANTS = ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#review-h', 'claude-opus-5-5#no-gstack');
|
||||
|
||||
describe('the Comparison declaration', () => {
|
||||
test('Comparison is a required section; the blank template fails and multi-line guidance never counts as content', () => {
|
||||
const t = preregTemplate('x');
|
||||
expect(t).toContain('## Comparison');
|
||||
expect(t).toContain('## No-gstack arm');
|
||||
const blank = checkPrereg(t);
|
||||
expect(blank.empty).toContain('Comparison');
|
||||
expect(checkPrereg(BASE + VARIANTS + '\n## Comparison\n<!-- stack: loaded\n more guidance -->\n').empty).toContain('Comparison');
|
||||
});
|
||||
|
||||
test('stack: none needs a why: line and passes without the arm; an unknown value fails', () => {
|
||||
const none = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration; no arm loads a skill stack\n');
|
||||
expect(none.ok).toBe(true);
|
||||
expect(none.comparison.stack).toBe('none');
|
||||
expect(none.no_gstack.required).toBe(false);
|
||||
const noWhy = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\n');
|
||||
expect(noWhy.ok).toBe(false);
|
||||
expect(noWhy.comparison.errors.join(';')).toContain('why:');
|
||||
const odd = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: maybe\n');
|
||||
expect(odd.ok).toBe(false);
|
||||
expect(odd.comparison.errors.join(';')).toContain('stack: maybe');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the No-gstack arm section', () => {
|
||||
test('a loaded comparison passes with a complete section and fails without it', () => {
|
||||
expect(checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK()).ok).toBe(true);
|
||||
const missing = checkPrereg(BASE + VARIANTS + LOADED);
|
||||
expect(missing.ok).toBe(false);
|
||||
expect(missing.no_gstack).toMatchObject({ required: true, ok: false });
|
||||
expect(missing.no_gstack.problems).toEqual(['missing section']);
|
||||
});
|
||||
|
||||
for (const key of ['arms', 'inputs', 'prompt', 'panel', 'report'] as const) {
|
||||
test(`a missing or blank ${key}: line fails naming it`, () => {
|
||||
for (const v of [undefined, ' ']) {
|
||||
const r = checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK({ [key]: v } as Record<string, string>));
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.no_gstack.problems).toContain(`missing ${key}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('every model that loads a stack needs its own <model>#no-gstack arm: a two-model panel needs two', () => {
|
||||
const panel = ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack');
|
||||
const one = checkPrereg(BASE + panel + LOADED + NO_GSTACK());
|
||||
expect(one.ok).toBe(false);
|
||||
expect(one.no_gstack.problems).toEqual(['no no-gstack arm for gpt-6.1-sol']);
|
||||
const both = checkPrereg(BASE + ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack', 'gpt-6.1-sol#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack, gpt-6.1-sol#no-gstack' }));
|
||||
expect(both.ok).toBe(true);
|
||||
});
|
||||
|
||||
test('a model panel under another skill stack (not gstack) is loaded too and needs the same arms', () => {
|
||||
const other = ARMS('claude-opus-5-5#other-stack', 'gpt-6.1-sol#other-stack');
|
||||
const r = checkPrereg(BASE + other + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack' }));
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.no_gstack.problems).toEqual(['claude-opus-5-5#no-gstack is not an Arms line', 'no no-gstack arm for claude-opus-5-5', 'no no-gstack arm for gpt-6.1-sol']);
|
||||
});
|
||||
|
||||
test('a listed arm must be an Arms line and a <model>#no-gstack identity, matched on whole identities', () => {
|
||||
const notListed = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g') + LOADED + NO_GSTACK());
|
||||
expect(notListed.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line');
|
||||
const prefix = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5#no-gstack' }));
|
||||
expect(prefix.no_gstack.problems).toContain('no no-gstack arm for claude-opus-5-5');
|
||||
const wrongLoadout = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#plain') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#plain' }));
|
||||
expect(wrongLoadout.no_gstack.problems).toContain('claude-opus-5-5#plain is not a <model>#no-gstack identity');
|
||||
const prose = checkPrereg(BASE + '\n## Arms\n- claude-opus-5-5#review-g, compared with claude-opus-5-5#no-gstack in prose\n' + LOADED + NO_GSTACK());
|
||||
expect(prose.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line');
|
||||
});
|
||||
|
||||
test('a table in Arms: the header row is not an arm', () => {
|
||||
const table = '\n## Arms\n| model | settings |\n|---|---|\n| claude-opus-5-5#review-g | default |\n| claude-opus-5-5#no-gstack | default |\n';
|
||||
expect(checkPrereg(BASE + table + LOADED + NO_GSTACK()).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('arm identities in pricing and the pilot', () => {
|
||||
test('armModel strips the loadout; arms and the dry run price <model>#no-gstack by its model', () => {
|
||||
expect(armModel('anthropic/claude-opus-5-5#no-gstack')).toBe('anthropic/claude-opus-5-5');
|
||||
expect(armModel('gpt-6.1-sol')).toBe('gpt-6.1-sol');
|
||||
const d = tmp();
|
||||
const arms = resolveArms({ cwd: d, explicit: ['claude-opus-5-5#no-gstack'] });
|
||||
expect(arms.errors).toEqual([]);
|
||||
expect(arms.arms[0]).toMatchObject({ model: 'claude-opus-5-5#no-gstack', priced: true });
|
||||
const rows = path.join(d, 'rows.jsonl');
|
||||
fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n');
|
||||
const dry = dryRun({ input: rows, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'] });
|
||||
expect(dry.checks.price_table.ok).toBe(true);
|
||||
expect(dry.per_item_usd['claude-opus-5-5#no-gstack']).toBe(dry.per_item_usd['claude-opus-5-5']);
|
||||
});
|
||||
|
||||
const passingDryRun = (d: string) => {
|
||||
const rows = path.join(d, 'rows.jsonl');
|
||||
fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n');
|
||||
const file = path.join(d, 'dry-run.json');
|
||||
const r = dryRun({ input: rows, arms: ['claude-opus-5-5'], retryPrompt: (fs.writeFileSync(path.join(d, 'retry.md'), 'try again\n'), path.join(d, 'retry.md')) });
|
||||
fs.writeFileSync(file, JSON.stringify(r));
|
||||
expect(r.pass, JSON.stringify(r.checks)).toBe(true);
|
||||
return file;
|
||||
};
|
||||
|
||||
test('the pilot reserves a variant and its no-gstack arm on the same model under distinct attempts; the total is the sum', () => {
|
||||
const d = tmp();
|
||||
const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 2, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'], dryRun: passingDryRun(d) });
|
||||
expect(r.error).toBeUndefined();
|
||||
const ids = r.record.attempts.map(a => a.attempt);
|
||||
expect(new Set(ids).size).toBe(4);
|
||||
const ledger = JSON.parse(fs.readFileSync(path.join(d, 'pilot', fs.readdirSync(path.join(d, 'pilot')).find(f => f.includes('spend'))!), 'utf8'));
|
||||
const reserved = JSON.stringify(ledger);
|
||||
for (const id of ids) expect(reserved).toContain(id);
|
||||
const each = r.record.attempts[0]!.estimate_usd;
|
||||
expect(r.record.attempts.every(a => a.estimate_usd === each)).toBe(true);
|
||||
});
|
||||
|
||||
test('a duplicate arm identity is refused before anything is reserved (EVAL_ARM_DUPLICATE)', () => {
|
||||
const d = tmp();
|
||||
const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms: ['claude-opus-5-5', 'claude-opus-5-5'], dryRun: passingDryRun(d) });
|
||||
expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE');
|
||||
expect(r.record.attempts).toEqual([]);
|
||||
});
|
||||
|
||||
test('distinct identities that share one ledger attempt name are refused before anything is reserved', () => {
|
||||
for (const arms of [['claude-opus-5-5#review/a', 'claude-opus-5-5#review_a'], ['gpt-6#1-sol', 'gpt-6.1-sol']]) {
|
||||
const d = tmp();
|
||||
const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms, dryRun: passingDryRun(d) });
|
||||
expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE');
|
||||
expect(r.error?.message).toContain(arms.join(' = '));
|
||||
expect(r.record.attempts).toEqual([]);
|
||||
expect(fs.existsSync(path.join(d, 'pilot'))).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-eval-plan prereg --check', () => {
|
||||
const run = (text: string) => {
|
||||
const d = tmp();
|
||||
const file = path.join(d, 'prereg.md');
|
||||
fs.writeFileSync(file, text);
|
||||
const r = spawnSync(BIN, ['prereg', '--check', file], { cwd: d, encoding: 'utf8', timeout: 60_000 });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
|
||||
test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => {
|
||||
const r = run(BASE + VARIANTS + LOADED);
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m);
|
||||
expect(r.err).toContain('missing section');
|
||||
expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)');
|
||||
expect(r.err).not.toContain('(PREREG_INCOMPLETE)');
|
||||
});
|
||||
|
||||
test('missing sections and arm problems both print, PREREG_INCOMPLETE first', () => {
|
||||
const r = run(VARIANTS + LOADED);
|
||||
expect(r.code).toBe(1);
|
||||
const a = r.err.indexOf('(PREREG_INCOMPLETE)'), b = r.err.indexOf('(PREREG_NO_GSTACK_ARM)');
|
||||
expect(a).toBeGreaterThanOrEqual(0);
|
||||
expect(b).toBeGreaterThan(a);
|
||||
});
|
||||
|
||||
test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => {
|
||||
const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK());
|
||||
expect(ok.code, ok.out + ok.err).toBe(0);
|
||||
expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m);
|
||||
const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n');
|
||||
expect(none.code, none.out + none.err).toBe(0);
|
||||
expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m);
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval
|
||||
import { capyHost, type CapyHost } from './helpers/capy-host';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n';
|
||||
const SUMMARIZER = [
|
||||
'touch ran.marker',
|
||||
'r=$(grep -c \'"kind":"refusal"\' "$1")',
|
||||
@@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']]));
|
||||
expect(r.code).toBe(3);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: ok present=5/5 diagnostics=2 file=');
|
||||
expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file=');
|
||||
expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`);
|
||||
expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)');
|
||||
expect(r.out).not.toContain('PREREG_DIAGNOSTICS:');
|
||||
@@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands');
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.ran).toBe(false);
|
||||
expect(r.out).toContain('PREREG: incomplete present=5/5 diagnostics=invalid');
|
||||
expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid');
|
||||
expect(r.err).toContain('diagnostics: row "| refusals | two |"');
|
||||
expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})');
|
||||
expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows');
|
||||
@@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => {
|
||||
test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => {
|
||||
const r = check(FILLED);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.out).toBe(`PREREG: ok present=5/5 file=${path.join(repo, 'evals', 'prereg.md')}\n`);
|
||||
expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`);
|
||||
expect(parsePreregDiagnostics(FILLED)).toBeNull();
|
||||
expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n<!-- none -->\n')).toBeNull();
|
||||
expect(preregTemplate()).toContain('## Diagnostics');
|
||||
|
||||
+1
@@ -101,6 +101,7 @@
|
||||
{ "file": "lib/result-codes-release.ts", "movedFrom": "lib/result-codes.ts (the release-path rows: ship queue, pregate timing, approval classes, state matrix; field list 2 wave B)" },
|
||||
{ "file": "lib/result-codes-capy.ts", "movedFrom": "lib/result-codes.ts (the Capy rows: approval record, Captain brief, PR screenshots; GSTA-61)" },
|
||||
{ "file": "lib/result-codes-budget.ts", "movedFrom": "lib/result-codes.ts (the /ship token budget and handoff rows; GSTA-152)" },
|
||||
{ "file": "lib/result-codes-eval.ts", "movedFrom": "lib/result-codes.ts (the /eval-plan preregistration rows and the no-gstack arm; GSTA-168)" },
|
||||
{ "file": "lib/ship-budget.ts", "movedFrom": "(new: /ship token budget ledger, meters and admission, GSTA-152)" },
|
||||
{ "file": "lib/ship-budget-cli.ts", "movedFrom": "(new: bin/gstack-ship-budget start/check/settle/close/resume, GSTA-152)" },
|
||||
{ "file": "lib/ship-handoff.ts", "movedFrom": "(new: /ship fresh-context handoff, verify, return and reconcile, GSTA-152)" },
|
||||
|
||||
@@ -27,7 +27,7 @@ const COPIED = [
|
||||
'autoplan/SKILL.md', // its frontmatter registers the autoplan hook the hooks-row test breaks
|
||||
'bin/gstack-launch-probe.sh',
|
||||
'bin/gstack-doctor-check.sh', 'bin/gstack-doctor-components.sh', 'bin/gstack-runtime-pins.ts', 'lib/runtime-pins.ts',
|
||||
'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', // the windows bash row
|
||||
'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', // the windows bash row
|
||||
];
|
||||
const bases: string[] = [];
|
||||
afterEach(() => { for (const b of bases.splice(0)) fs.rmSync(b, { recursive: true, force: true }); });
|
||||
|
||||
@@ -70,7 +70,7 @@ function scaffoldFixture(workDir: string): { workDir: string; stateRoot: string;
|
||||
// These imports are resolved relative to the copied install, not the source checkout.
|
||||
const libDir = path.join(workDir, 'lib');
|
||||
fs.mkdirSync(libDir, { recursive: true });
|
||||
for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts']) {
|
||||
for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts', 'result-codes-eval.ts']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'lib', file), path.join(libDir, file));
|
||||
}
|
||||
|
||||
|
||||
@@ -50,7 +50,8 @@ describe('preregistration', () => {
|
||||
expect(check.missing).toEqual([]);
|
||||
expect(check.empty.length).toBe(PREREG_SECTIONS.length);
|
||||
expect(check.ok).toBe(false);
|
||||
const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in');
|
||||
expect(PREREG_SECTIONS).toContain('Comparison');
|
||||
const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack');
|
||||
expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] });
|
||||
expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]);
|
||||
});
|
||||
|
||||
@@ -33,6 +33,7 @@ const HOOK_FILES = [
|
||||
'lib/result-codes-release.ts',
|
||||
'lib/result-codes-capy.ts',
|
||||
'lib/result-codes-budget.ts',
|
||||
'lib/result-codes-eval.ts',
|
||||
];
|
||||
const SLUG = 'stop-hook-slug-test';
|
||||
const BUDGET_MS = 2000;
|
||||
|
||||
Reference in new issue
Block a user