mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-19 14:44:44 +01:00
feat(eval-plan): results --check refuses a results table without its no-gstack rows
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
0cd2b4a9f9
commit
74d5588748
10 files changed
+216
-5
No files matched your search
@@ -347,6 +347,7 @@ gstack_result_anchor() {
|
||||
PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;;
|
||||
PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;;
|
||||
PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;;
|
||||
EVAL_REPORT_NO_GSTACK_ROW) printf '%s' 'eval-report-no-gstack-row' ;;
|
||||
EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
|
||||
+20
-2
@@ -10,6 +10,7 @@
|
||||
* gstack-eval-plan pilot settle --out <dir> --attempt <id> --usd <n|unknown>
|
||||
* gstack-eval-plan pilot release --out <dir> --attempt <id>
|
||||
* gstack-eval-plan pilot report --out <dir> --total <items> [--workers W] [--json]
|
||||
* gstack-eval-plan results --check <prereg> <results.md> [--json]
|
||||
* gstack-eval-plan price-gate --gate-file <file> | --gate "<exact gate command>" [--plan-file <file>] [--base <branch>] [--cap-usd N]
|
||||
* [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out <dir>] [--json]
|
||||
*
|
||||
@@ -26,12 +27,13 @@
|
||||
* price-gate lint every arm's model against the base branch's
|
||||
* .gstack/eval-models.json (docs/eval-models.md; lib/eval-models.ts). Exit 0 ok, 1 fail (incomplete
|
||||
* prereg, failed check, exceeded cap, unpriced or stale gate), 2 usage, 3 refused.
|
||||
* Logic: lib/eval-plan.ts.
|
||||
* Logic: lib/eval-plan.ts; `results --check` in lib/eval-plan-report.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan';
|
||||
import { checkResults } from '../lib/eval-plan-report';
|
||||
import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
@@ -41,7 +43,8 @@ import { SpendCapExceeded } from '../lib/spend-ledger';
|
||||
const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, priced pilot (eval-plan/SKILL.md)
|
||||
|
||||
prereg --check <file> [--allow-repo-commands] [--json]
|
||||
the five sections exist and are filled: Bars, Arms, Decoys, Stop rules, Held-out exposure;
|
||||
the required sections exist and are filled: Bars, Arms, Comparison, Decoys, Stop rules,
|
||||
Held-out exposure, and No-gstack arm when the Comparison says stack: loaded;
|
||||
an optional Diagnostics section's summarizer runs on its synthetic fixture only with
|
||||
--allow-repo-commands (else it is listed and exit 3) and every count must match
|
||||
prereg --init <file> [--title <t>] write the template
|
||||
@@ -53,6 +56,9 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri
|
||||
pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3]
|
||||
pilot settle --out dir --attempt id --usd <n|unknown> pilot release --out dir --attempt id
|
||||
pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask
|
||||
results --check <prereg> <results.md> [--json]
|
||||
one table in the results file has a filled row for every Arms identity, each no-gstack arm
|
||||
beside its variants; reads text only (docs/TESTING_INTERNALS.md#no-gstack-arm)
|
||||
price-gate --gate-file <file> | --gate "<exact gate command>" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced "<reason>"] [--out dir] [--json]
|
||||
price a paid gate from its GSTACK_EVAL_CALLS v1 call-plan file (--plan-file, else the base
|
||||
policy's price_file at HEAD); checks the gate/head/input binding, executes nothing, writes a
|
||||
@@ -129,6 +135,17 @@ function doPrereg(rest: string[]): number {
|
||||
return EXIT.ok;
|
||||
}
|
||||
|
||||
function doResults(rest: string[]): number {
|
||||
const prereg = flag(rest, '--check');
|
||||
const results = rest.filter(a => !a.startsWith('--') && a !== prereg)[0];
|
||||
if (!prereg || !results || !fs.existsSync(prereg) || !fs.existsSync(results)) return usage('results needs --check <existing prereg> <existing results file>');
|
||||
const r = checkResults(fs.readFileSync(prereg, 'utf8'), fs.readFileSync(results, 'utf8'));
|
||||
if (rest.includes('--json')) process.stdout.write(JSON.stringify({ prereg, results, ...r }) + '\n');
|
||||
else process.stdout.write(`RESULTS: ${r.ok ? 'ok' : 'incomplete'} no_gstack_rows=${r.no_gstack.filter(id => r.found.includes(id)).length}/${r.no_gstack.length} rows=${r.found.length}/${r.required.length} file=${results}\n`);
|
||||
if (!r.ok) { process.stderr.write(resultLine('gstack-eval-plan results', 'EVAL_REPORT_NO_GSTACK_ROW', r.problems.join('; ')) + '\n'); return EXIT.fail; }
|
||||
return EXIT.ok;
|
||||
}
|
||||
|
||||
function doArms(rest: string[]): number {
|
||||
const tokens = { input: num(rest, '--input-tokens', DEFAULT_TOKENS.input)!, output: num(rest, '--output-tokens', DEFAULT_TOKENS.output)! };
|
||||
const scope = scopeOf(rest);
|
||||
@@ -242,6 +259,7 @@ function main(argv: string[]): number {
|
||||
if (cmd === 'dry-run') return doDryRun(rest);
|
||||
if (cmd === 'pilot') return doPilot(rest);
|
||||
if (cmd === 'price-gate') return doPriceGate(rest);
|
||||
if (cmd === 'results') return doResults(rest);
|
||||
return usage(`unknown command ${cmd}`);
|
||||
} catch (e: any) {
|
||||
if (e instanceof SpendCapExceeded) { process.stderr.write(resultLine(`gstack-eval-plan ${cmd}`, 'EVAL_BUDGET_EXHAUSTED', e.message) + '\n'); return EXIT.fail; }
|
||||
|
||||
@@ -1676,9 +1676,10 @@ own attempts. The arm gets:
|
||||
|
||||
**Reporting.** The no-gstack result is one row beside the variants in every
|
||||
results table and every verdict, including when it wins. A comparison that
|
||||
drops the row has not reported its result. The prereg check cannot see the
|
||||
report, so the `report:` line states where the row goes and the reviewer of
|
||||
the result holds the author to it.
|
||||
drops the row has not reported its result. `gstack-eval-plan results --check
|
||||
<prereg> <results.md>` enforces it: one table in the results file must hold a
|
||||
filled row for every Arms identity (`EVAL_REPORT_NO_GSTACK_ROW` otherwise), and
|
||||
no number is published before it passes.
|
||||
|
||||
**Precedent and its gaps.** `test/helpers/arm-benchmark-harness.ts` already has
|
||||
a `without-skill` arm beside `with-skill`, and it is the closest existing
|
||||
|
||||
@@ -3672,6 +3672,21 @@ goes beside the variants. When no arm loads a skill stack, write
|
||||
`stack: none` with a `why:` line instead. The rule is in
|
||||
[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm).
|
||||
|
||||
<a id="eval-report-no-gstack-row"></a>
|
||||
### `EVAL_REPORT_NO_GSTACK_ROW`
|
||||
|
||||
**Meaning.** `gstack-eval-plan results --check <prereg> <results>` found no
|
||||
single markdown table in the results file with a filled row for every arm
|
||||
identity the preregistration's Arms section lists. Usually the
|
||||
`<model>#no-gstack` row is missing, sits in a separate table from its
|
||||
variants, or holds only placeholders (`-`, `n/a`, `tbd`). It also fires when
|
||||
the preregistration itself fails `prereg --check`.
|
||||
|
||||
**Fix.** Put one row per arm identity, each no-gstack arm beside its variants,
|
||||
in one table, and rerun the check. Publish no number before it prints
|
||||
`RESULTS: ok`. A preregistration with `stack: none` needs no row. The rule is
|
||||
in [docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm).
|
||||
|
||||
<a id="eval-arm-duplicate"></a>
|
||||
### `EVAL_ARM_DUPLICATE`
|
||||
|
||||
|
||||
@@ -307,6 +307,20 @@ Under `SESSION_KIND: unattended` write the question as an `approval` gate item t
|
||||
The full run, when approved, keeps settling every attempt into the same ledger and reprints
|
||||
the spend table at each gate; `budget_exhausted` stops it.
|
||||
|
||||
## Step 5: Results beside the no-gstack arm
|
||||
|
||||
Write the run's results as one markdown table in a results file, one row per arm identity
|
||||
from the Arms section, each `<model>#no-gstack` row beside its variants. Before any number
|
||||
leaves the run (a report, a READY block, a PR body), check it:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan results --check <prereg-path> <results-path>
|
||||
```
|
||||
|
||||
`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm
|
||||
whose filled row is missing from the table that holds the others; add it and rerun. A
|
||||
preregistration with `stack: none` needs no row.
|
||||
|
||||
## Tier 3, after P7 (not in this skill yet)
|
||||
|
||||
Each of these waits for the adoption number in the multi-agent wave plan (P7); one line each so the shape is known:
|
||||
|
||||
@@ -222,6 +222,20 @@ Under `SESSION_KIND: unattended` write the question as an `approval` gate item t
|
||||
The full run, when approved, keeps settling every attempt into the same ledger and reprints
|
||||
the spend table at each gate; `budget_exhausted` stops it.
|
||||
|
||||
## Step 5: Results beside the no-gstack arm
|
||||
|
||||
Write the run's results as one markdown table in a results file, one row per arm identity
|
||||
from the Arms section, each `<model>#no-gstack` row beside its variants. Before any number
|
||||
leaves the run (a report, a READY block, a PR body), check it:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-eval-plan results --check <prereg-path> <results-path>
|
||||
```
|
||||
|
||||
`RESULTS: ok` lets the numbers go out. `EVAL_REPORT_NO_GSTACK_ROW` (exit 1) names the arm
|
||||
whose filled row is missing from the table that holds the others; add it and rerun. A
|
||||
preregistration with `stack: none` needs no row.
|
||||
|
||||
## Tier 3, after P7 (not in this skill yet)
|
||||
|
||||
Each of these waits for the adoption number in the multi-agent wave plan (P7); one line each so the shape is known:
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* eval-plan-report: the results check behind `gstack-eval-plan results --check`.
|
||||
* A preregistration that says `stack: loaded` names its no-gstack arms; its
|
||||
* results file must hold one markdown table with a filled row for every Arms
|
||||
* identity, so each no-gstack row sits beside its variants
|
||||
* (docs/TESTING_INTERNALS.md#no-gstack-arm). Reads text only; executes nothing.
|
||||
*/
|
||||
import { armIdentities, checkPrereg } from './eval-plan';
|
||||
|
||||
export interface ResultsCheck { ok: boolean; required: string[]; no_gstack: string[]; found: string[]; missing: string[]; problems: string[] }
|
||||
|
||||
const PLACEHOLDER = /^(?:|-+|—|–|n\/a|tbd|todo|\?)$/i;
|
||||
const cellsOf = (line: string) => line.trim().replace(/^\|/, '').replace(/\|$/, '').split('|').map(c => c.trim());
|
||||
const separator = (line: string | undefined) => !!line && /^\s*\|[\s:|-]+\|?\s*$/.test(line) && line.includes('-');
|
||||
|
||||
/** The filled-row identities of each markdown table: first cell, backticks stripped, header and separator rows skipped. */
|
||||
export function resultTables(text: string): Array<Set<string>> {
|
||||
const lines = text.split('\n');
|
||||
const tables: Array<Set<string>> = [];
|
||||
let current: Set<string> | null = null;
|
||||
lines.forEach((line, i) => {
|
||||
if (!/^\s*\|/.test(line)) { current = null; return; }
|
||||
if (!current) { current = new Set(); tables.push(current); }
|
||||
if (separator(line) || separator(lines[i + 1])) return;
|
||||
const [first, ...rest] = cellsOf(line);
|
||||
const id = (first ?? '').replace(/`/g, '').trim();
|
||||
if (id && rest.some(c => !PLACEHOLDER.test(c.replace(/`/g, '').trim()))) current.add(id);
|
||||
});
|
||||
return tables;
|
||||
}
|
||||
|
||||
export function checkResults(preregText: string, resultsText: string): ResultsCheck {
|
||||
const prereg = checkPrereg(preregText);
|
||||
if (prereg.comparison.stack === 'none') return { ok: true, required: [], no_gstack: [], found: [], missing: [], problems: [] };
|
||||
const required = [...new Set(armIdentities(preregText))];
|
||||
const noGstack = prereg.no_gstack.arms;
|
||||
const problems: string[] = [];
|
||||
if (!prereg.ok) problems.push('the preregistration does not pass prereg --check');
|
||||
const tables = resultTables(resultsText);
|
||||
const best = tables.reduce<Set<string>>((b, t) => (required.filter(id => t.has(id)).length > required.filter(id => b.has(id)).length ? t : b), new Set());
|
||||
const found = required.filter(id => best.has(id));
|
||||
const missing = required.filter(id => !best.has(id));
|
||||
if (!tables.length) problems.push('no markdown table in the results file');
|
||||
else for (const id of missing) problems.push(`no filled row for ${id} in the table that holds the other arms`);
|
||||
return { ok: problems.length === 0, required, no_gstack: noGstack, found, missing, problems };
|
||||
}
|
||||
@@ -25,6 +25,11 @@ export const EVAL_RESULT_CODES = {
|
||||
summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own <model>#no-gstack Arms line',
|
||||
fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each <model>#no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line',
|
||||
},
|
||||
EVAL_REPORT_NO_GSTACK_ROW: {
|
||||
anchor: 'eval-report-no-gstack-row',
|
||||
summary: 'the results file has no single table with a filled row for every arm the preregistration lists, so a no-gstack row is missing or not beside its variants',
|
||||
fix: 'put one row per arm identity (each <model>#no-gstack beside its variants) in one markdown table of the results file, then rerun `gstack-eval-plan results --check`; publish no number before it says ok',
|
||||
},
|
||||
EVAL_ARM_DUPLICATE: {
|
||||
anchor: 'eval-arm-duplicate',
|
||||
summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts',
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* GST-471: `gstack-eval-plan results --check`. A loaded preregistration's
|
||||
* results file must hold one markdown table with a filled row for every Arms
|
||||
* identity, so each no-gstack row sits beside its variants.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { checkResults } from '../lib/eval-plan-report';
|
||||
|
||||
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
|
||||
const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n');
|
||||
const PREREG = BASE + `
|
||||
## Arms
|
||||
- claude-opus-5-5#review-g default settings
|
||||
- claude-opus-5-5#no-gstack default settings
|
||||
|
||||
## Comparison
|
||||
stack: loaded
|
||||
|
||||
## No-gstack arm
|
||||
arms: claude-opus-5-5#no-gstack
|
||||
inputs: the same rows
|
||||
prompt: the task only
|
||||
panel: rule, 3 trials, same judge
|
||||
report: results.md, one row beside the variant
|
||||
`;
|
||||
const NONE = BASE + '\n## Arms\n- claude-opus-5-5 default\n\n## Comparison\nstack: none\nwhy: scorer calibration\n';
|
||||
const table = (...rows: string[]) => ['| arm | passed | cost |', '|---|---|---|', ...rows].join('\n') + '\n';
|
||||
|
||||
describe('checkResults', () => {
|
||||
test('one table with a filled row per arm passes; backticks and the header row are handled', () => {
|
||||
const r = checkResults(PREREG, '# Results\n\n' + table('| `claude-opus-5-5#review-g` | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
|
||||
expect(r).toMatchObject({ ok: true, no_gstack: ['claude-opus-5-5#no-gstack'], missing: [] });
|
||||
});
|
||||
|
||||
test('a missing no-gstack row fails naming it', () => {
|
||||
const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |'));
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.problems).toEqual(['no filled row for claude-opus-5-5#no-gstack in the table that holds the other arms']);
|
||||
});
|
||||
|
||||
test('placeholder cells do not count as a filled row', () => {
|
||||
for (const cell of ['', '-', '—', 'n/a', 'TBD']) {
|
||||
const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', `| claude-opus-5-5#no-gstack | ${cell} | ${cell} |`));
|
||||
expect(r.missing, JSON.stringify(cell)).toEqual(['claude-opus-5-5#no-gstack']);
|
||||
}
|
||||
});
|
||||
|
||||
test('the no-gstack row must sit in the same table as its variants; prose mentions never count', () => {
|
||||
const split = table('| claude-opus-5-5#review-g | 7/9 | $1.20 |') + '\nThe claude-opus-5-5#no-gstack arm passed 5/9.\n\n' + table('| claude-opus-5-5#no-gstack | 5/9 | $0.80 |');
|
||||
expect(checkResults(PREREG, split).ok).toBe(false);
|
||||
expect(checkResults(PREREG, 'claude-opus-5-5#review-g 7/9, claude-opus-5-5#no-gstack 5/9\n').problems).toEqual(['no markdown table in the results file']);
|
||||
});
|
||||
|
||||
test('identities match whole: a prefix model does not stand in for the arm', () => {
|
||||
const r = checkResults(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5#no-gstack | 5/9 | $0.80 |'));
|
||||
expect(r.missing).toEqual(['claude-opus-5-5#no-gstack']);
|
||||
});
|
||||
|
||||
test('stack: none needs no row; a preregistration that fails prereg --check fails here too', () => {
|
||||
expect(checkResults(NONE, 'no table at all\n').ok).toBe(true);
|
||||
const broken = PREREG.replace(/## No-gstack arm[\s\S]*$/, '');
|
||||
const r = checkResults(broken, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
|
||||
expect(r.problems).toEqual(['the preregistration does not pass prereg --check']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-eval-plan results --check', () => {
|
||||
const run = (prereg: string, results: string) => {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-results-check-'));
|
||||
dirs.push(d);
|
||||
fs.writeFileSync(path.join(d, 'prereg.md'), prereg);
|
||||
fs.writeFileSync(path.join(d, 'results.md'), results);
|
||||
const r = spawnSync(BIN, ['results', '--check', 'prereg.md', 'results.md'], { cwd: d, encoding: 'utf8', timeout: 60_000 });
|
||||
return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' };
|
||||
};
|
||||
|
||||
test('missing row: RESULTS: incomplete, EVAL_REPORT_NO_GSTACK_ROW, exit 1', () => {
|
||||
const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |'));
|
||||
expect(r.code).toBe(1);
|
||||
expect(r.out).toBe('RESULTS: incomplete no_gstack_rows=0/1 rows=1/2 file=results.md\n');
|
||||
expect(r.err).toContain('(EVAL_REPORT_NO_GSTACK_ROW)');
|
||||
});
|
||||
|
||||
test('complete: RESULTS: ok, exit 0', () => {
|
||||
const r = run(PREREG, table('| claude-opus-5-5#review-g | 7/9 | $1.20 |', '| claude-opus-5-5#no-gstack | 5/9 | $0.80 |'));
|
||||
expect(r.code, r.err).toBe(0);
|
||||
expect(r.out).toBe('RESULTS: ok no_gstack_rows=1/1 rows=2/2 file=results.md\n');
|
||||
});
|
||||
});
|
||||
+1
@@ -34,6 +34,7 @@
|
||||
{ "file": "lib/design-lint/rules/index.ts", "movedFrom": "(new: the bundled design lint, GSTA-105; ported from hallmark sloplint.mjs)" },
|
||||
{ "file": "lib/design-lint/rules/copy.ts", "movedFrom": "(new: the bundled design lint, GSTA-105; ported from hallmark sloplint.mjs)" },
|
||||
{ "file": "lib/pricing.ts", "movedFrom": "test/helpers/pricing.ts (PRICING, estimateCostUsd; plan E1 moved the table under lib/ so bin/gstack-eval-plan can price arms)" },
|
||||
{ "file": "lib/eval-plan-report.ts", "movedFrom": "(new: the results check behind gstack-eval-plan results --check, the no-gstack row beside its variants; GST-471)" },
|
||||
{ "file": "lib/eval-plan.ts", "movedFrom": "(new: preregistration check, priced arms, $0 dry run, priced pilot ledger behind bin/gstack-eval-plan, plan E1)" },
|
||||
{ "file": "lib/lane-ownership.ts", "movedFrom": "(new: lane ownership check behind bin/gstack-lane-check, plan E2)" },
|
||||
{ "file": "lib/coordinator-contract.ts", "movedFrom": "(new: the inherited coordinator contract text, plan E2)" },
|
||||
|
||||
Reference in new issue
Block a user